diff --git a/.agents/skills/e2e-tests/SKILL.md b/.agents/skills/e2e-tests/SKILL.md index 9989cd7b8..4dae66340 100644 --- a/.agents/skills/e2e-tests/SKILL.md +++ b/.agents/skills/e2e-tests/SKILL.md @@ -60,6 +60,7 @@ Cassettes mock provider HTTP responses (OpenAI, Anthropic, ...) so external-prov - Run every scenario through `withScenarioHarness(...)`. - Keep reusable logic in `e2e/helpers/`. Keep one-off fixtures and scenario-specific files inside the scenario directory. - Snapshot stable contracts, not raw noise. Use `normalizeForSnapshot(...)` before inline snapshots and `formatJsonFileSnapshot(...)` plus file snapshots for larger payloads or version matrices. +- For span-tree snapshots, use `matchSpanTreeSnapshot(...)`. It writes and asserts paired `.span-tree.json` and `.span-tree.txt` files from the same normalized span tree. The JSON file is the structural contract that is easiest to parse mechanically; the TXT file is the ASCII tree that is easiest to review by eye. Keep them in sync by updating both through the e2e update/record commands, and never hand-edit only one side of the pair. - When a scenario family already has `assertions.ts`, keep version- or provider-specific test setup in `scenario.test.ts` and reuse the shared assertions file. - Keep the CI e2e summary up to date. If a scenario version matrix or `variantKey` changes, update `e2e/config/pr-comment-scenarios.json` in the same change and follow the established pattern used by other versioned scenarios: one summary row per version, not separate wrapped/auto rows unless that pattern already exists for the scenario family. - Run new or updated scenarios three times in a row before considering snapshots stable. diff --git a/AGENTS.md b/AGENTS.md index 7efa16332..f4d2285c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,9 +59,14 @@ Each scenario runs the SDK in a subprocess against a mock Braintrust server and ```bash pnpm run test:e2e # Run all e2e scenarios (from repo root) +pnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes pnpm run test:e2e:record # Re-record provider cassettes and update snapshots ``` +When adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output. + +Span-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair. + **From repo root:** ```bash diff --git a/e2e/README.md b/e2e/README.md index 04bc2a7fb..c02cc63bf 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -34,7 +34,7 @@ Any extra files needed only by one scenario stay in that scenario folder. Anythi - `scenario-installer.ts` - Installs optional scenario-local dependencies from a colocated `package.json` into a shared cache and links them into prepared scenario copies. - `mock-braintrust-server.ts` - Captures requests, merged log payloads, and parsed span-like events. - `normalize.ts` - Makes snapshots deterministic by normalizing ids, timestamps, paths, and mock-server URLs. -- `trace-selectors.ts` / `trace-summary.ts` - Helpers for finding spans and snapshotting only the relevant shape. +- `trace-selectors.ts` / `span-tree.ts` / `trace-summary.ts` - Helpers for finding spans and snapshotting stable, human-readable trace trees. - `scenario-runtime.ts` - Shared runtime utilities used by scenario entrypoints. - `openai.ts` - Shared scenario lists and assertions for OpenAI wrapper and hook coverage across v4/v5/v6. - `wrapper-contract.ts` - Helpers for snapshotting wrapper span contracts and filtering payload rows by root span id. @@ -66,7 +66,7 @@ The main utilities you'll use in test files: - `events()`, `payloads()`, `requestCursor()`, `requestsAfter()` - Lower-level access for ingestion payloads and HTTP request flow assertions. - `testRunId` - Useful when a scenario or assertion needs the exact run marker. -Use `normalizeForSnapshot(...)` before snapshotting. It replaces timestamps and ids with stable tokens and strips machine-specific paths and localhost ports. +Prefer `matchSpanTreeSnapshot(...)` for span snapshots. It asserts both a structural `.span-tree.json` snapshot and a human-readable `.span-tree.txt` tree beside it. Both files are generated from the same normalized span tree and include stable span attributes, input, output, expected values, scores, tags, metadata, metrics, and errors. Use `normalizeForSnapshot(...)` for non-span JSON snapshots; it replaces timestamps and ids with stable tokens and strips machine-specific paths and localhost ports. ### Provider scenario cassettes @@ -79,7 +79,7 @@ Wrapper scenarios often create a root span with `testRunId` metadata and then le - Use `events()` rather than `testRunEvents()` to inspect the full trace tree. - Find the scenario root span first. - Scope raw payload snapshots by `root_span_id` using `payloadRowsForRootSpan(...)`. -- Pair a normalized `span-events` snapshot with a normalized `log-payloads` snapshot. +- Prefer normalized span-tree snapshots from `matchSpanTreeSnapshot(...)`. The `.json` sibling is the structural contract, and the `.txt` sibling is the ASCII tree for review; both are asserted and should be updated together. - If the wrapper has an explicit support matrix, reuse one shared test across version-specific scenario entries instead of duplicating the assertions. The AI SDK wrapper scenario uses this for supported v3-v6 package combinations. ### Runner-wrapper scenario pattern @@ -122,6 +122,7 @@ Scenario-local manifests are optional and should stay slim. They are only for sc ```bash pnpm run test:e2e # Run all e2e tests +pnpm run test:e2e:update # Update snapshots in cassette replay mode pnpm run test:e2e:record # Re-record provider cassettes and update snapshots pnpm run test:e2e:record -- # Re-record one scenario from the repo root pnpm run test:e2e:canary # Run canary e2e tests diff --git a/e2e/helpers/normalize.ts b/e2e/helpers/normalize.ts index 501380d64..02987f623 100644 --- a/e2e/helpers/normalize.ts +++ b/e2e/helpers/normalize.ts @@ -20,12 +20,23 @@ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const UUID_SUBSTRING_REGEX = /[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/gi; -const TIME_KEYS = new Set(["created", "date", "start", "end"]); +const TIME_KEYS = new Set([ + "completed_at", + "created", + "created_at", + "date", + "end", + "expires_at", + "start", + "started_at", + "updated_at", +]); const SPAN_ID_KEYS = new Set(["id", "span_id", "root_span_id"]); const ZERO_NUMBER_KEYS = new Set([ "avgLogprobs", "caller_lineno", "duration", + "github_copilot.context_window.current", "time_to_first_token", ]); const XACT_VERSION_KEYS = new Set([ @@ -46,7 +57,13 @@ const DYNAMIC_HEADER_KEYS = new Set([ "x-ratelimit-reset-tokens", "x-request-id", ]); -const PROVIDER_ID_KEYS = new Set(["itemId", "responseId", "toolCallId"]); +const PROVIDER_ID_KEYS = new Set([ + "agentId", + "claude_agent_sdk.task_id", + "itemId", + "responseId", + "toolCallId", +]); const PROJECT_ID_KEYS = new Set(["project_id", "projectId"]); const PROJECT_NAME_KEYS = new Set(["project_name", "projectName"]); const HELPERS_DIR = path.dirname(fileURLToPath(import.meta.url)); @@ -219,7 +236,12 @@ function normalizeValue( } if (typeof value === "number") { - if (currentKey && ZERO_NUMBER_KEYS.has(currentKey)) { + if ( + currentKey && + (ZERO_NUMBER_KEYS.has(currentKey) || + currentKey.endsWith("_ms") || + currentKey.endsWith("Ms")) + ) { return 0; } if (currentKey && TIME_KEYS.has(currentKey)) { @@ -240,6 +262,14 @@ function normalizeValue( return normalizeCallerFilename(value); } + if (currentKey === "openai_codex.working_directory") { + const normalizedPath = value.replace(/\\/g, "/"); + const match = normalizedPath.match( + /\/braintrust-codex-e2e-[^/]+\/([^/]+)$/, + ); + return match ? `/braintrust-codex-e2e/${match[1]}` : ""; + } + if (currentKey === "_xact_id") { return tokenFor(tokenMaps.xacts, value, "xact"); } diff --git a/e2e/helpers/span-tree.ts b/e2e/helpers/span-tree.ts new file mode 100644 index 000000000..331a9d34d --- /dev/null +++ b/e2e/helpers/span-tree.ts @@ -0,0 +1,383 @@ +import { matchFileSnapshot } from "./file-snapshot"; +import type { CapturedLogEvent } from "./mock-braintrust-server"; +import { normalizeForSnapshot, type Json } from "./normalize"; + +export type SpanTreeFields = Record; + +export type SpanTreeEntry = { + event: CapturedLogEvent; + fields?: SpanTreeFields; + name?: string; +}; + +type NormalizedEntry = { + children: NormalizedEntry[]; + event: CapturedLogEvent; + fields: SpanTreeFields; + firstSeen: number; + id: string; + name: string; +}; + +type SpanTreeJsonNode = { + children: SpanTreeJsonNode[]; + name: string; + type?: string; +} & Record; + +const FIELD_ORDER = [ + "span_attributes", + "input", + "output", + "expected", + "scores", + "tags", + "metadata", + "metrics", + "error", +]; + +const FIELD_LABELS = new Map([["span_attributes", "attributes"]]); +const OMITTED_SPAN_ATTRIBUTE_KEYS = new Set(["name", "type", "exec_counter"]); +const OMITTED_METRIC_KEYS = new Set(["start", "end"]); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function jsonClone(value: unknown): Json | undefined { + if (value === undefined) { + return undefined; + } + + return JSON.parse(JSON.stringify(value)) as Json; +} + +function normalizeJson(value: unknown): Json | undefined { + const cloned = jsonClone(value); + return cloned === undefined ? undefined : normalizeForSnapshot(cloned); +} + +function sortJsonKeys(value: Json): Json { + if (Array.isArray(value)) { + return value.map((entry) => sortJsonKeys(entry as Json)); + } + + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, sortJsonKeys(value[key] as Json)]), + ) as Json; + } + + return value; +} + +function compactRecord( + value: unknown, + omittedKeys: Set = new Set(), +): Record | undefined { + if (!isRecord(value)) { + return undefined; + } + + const compacted = Object.fromEntries( + Object.entries(value).filter( + ([key, entry]) => !omittedKeys.has(key) && entry !== undefined, + ), + ); + return Object.keys(compacted).length > 0 ? compacted : undefined; +} + +function stableMetrics(value: unknown): Record | undefined { + const metrics = compactRecord(value, OMITTED_METRIC_KEYS); + if (!metrics) { + return undefined; + } + + return Object.fromEntries( + Object.entries(metrics).map(([key, entry]) => [ + key, + typeof entry === "number" && + (key === "duration" || + key === "time_to_first_token" || + key.endsWith("_ms") || + key.includes("duration")) + ? 0 + : entry, + ]), + ); +} + +export function spanTreeFields(event: CapturedLogEvent): SpanTreeFields { + return { + span_attributes: compactRecord( + event.row.span_attributes, + OMITTED_SPAN_ATTRIBUTE_KEYS, + ), + input: event.input, + output: event.output, + expected: event.expected, + scores: event.scores, + tags: event.row.tags, + metadata: compactRecord(event.metadata), + metrics: stableMetrics(event.metrics), + error: event.row.error, + }; +} + +function toSpanTreeEntry( + input: CapturedLogEvent | SpanTreeEntry, +): SpanTreeEntry { + return "event" in input ? input : { event: input }; +} + +function entryId(entry: SpanTreeEntry, index: number): string { + return ( + entry.event.span.id ?? + (typeof entry.event.row.id === "string" + ? `row:${entry.event.row.id}` + : `index:${index}`) + ); +} + +function displayName(entry: SpanTreeEntry): string { + return entry.name ?? entry.event.span.name ?? ""; +} + +function sortedFieldKeys(fields: Record): string[] { + return Object.keys(fields).sort((left, right) => { + const leftOrder = FIELD_ORDER.indexOf(left); + const rightOrder = FIELD_ORDER.indexOf(right); + + if (leftOrder !== -1 || rightOrder !== -1) { + return ( + (leftOrder === -1 ? Number.MAX_SAFE_INTEGER : leftOrder) - + (rightOrder === -1 ? Number.MAX_SAFE_INTEGER : rightOrder) + ); + } + + return left.localeCompare(right); + }); +} + +function fieldLabel(key: string): string { + return FIELD_LABELS.get(key) ?? key; +} + +function shouldRenderField(value: Json | undefined): value is Json { + if (value === undefined) { + return false; + } + + if (Array.isArray(value)) { + return value.length > 0; + } + + if (value && typeof value === "object") { + return Object.keys(value).length > 0; + } + + return true; +} + +function normalizeFields(fields: SpanTreeFields): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(fields)) { + const normalizedValue = normalizeJson(value); + if (shouldRenderField(normalizedValue)) { + normalized[key] = sortJsonKeys(normalizedValue); + } + } + + return normalized; +} + +function formatFieldBlock( + label: string, + value: Json, + indent: string, +): string[] { + const lines = JSON.stringify(value, null, 2).split("\n"); + return [ + `${indent}${label}: ${lines[0]}`, + ...lines.slice(1).map((line) => `${indent}${line}`), + ]; +} + +function buildEntries( + inputs: readonly (CapturedLogEvent | SpanTreeEntry)[], +): NormalizedEntry[] { + const entriesById = new Map(); + + inputs.forEach((input, index) => { + const entry = toSpanTreeEntry(input); + const id = entryId(entry, index); + const existing = entriesById.get(id); + const normalized: NormalizedEntry = { + children: existing?.children ?? [], + event: entry.event, + fields: entry.fields ?? spanTreeFields(entry.event), + firstSeen: existing?.firstSeen ?? index, + id, + name: displayName(entry), + }; + entriesById.set(id, normalized); + }); + + const entries = [...entriesById.values()]; + const roots: NormalizedEntry[] = []; + + for (const entry of entries) { + const parentCandidates = entry.event.span.parentIds + .map((parentId) => entriesById.get(parentId)) + .filter( + (candidate): candidate is NormalizedEntry => candidate !== undefined, + ); + const childStart = + typeof entry.event.metrics?.start === "number" + ? entry.event.metrics.start + : undefined; + const parent = parentCandidates.reduce( + (best, candidate) => { + const candidateStart = + typeof candidate.event.metrics?.start === "number" + ? candidate.event.metrics.start + : undefined; + if ( + childStart !== undefined && + candidateStart !== undefined && + candidateStart > childStart + ) { + return best; + } + if (!best) { + return candidate; + } + + const bestStart = + typeof best.event.metrics?.start === "number" + ? best.event.metrics.start + : undefined; + if (candidateStart !== undefined && bestStart === undefined) { + return candidate; + } + if ( + candidateStart !== undefined && + bestStart !== undefined && + candidateStart !== bestStart + ) { + return candidateStart > bestStart ? candidate : best; + } + return candidate.firstSeen > best.firstSeen ? candidate : best; + }, + undefined, + ); + + if (parent) { + parent.children.push(entry); + } else { + roots.push(entry); + } + } + + const sortEntries = (items: NormalizedEntry[]) => { + items.sort((left, right) => left.firstSeen - right.firstSeen); + for (const item of items) { + sortEntries(item.children); + } + }; + sortEntries(roots); + + return roots; +} + +function renderEntry( + entry: NormalizedEntry, + prefix: string, + isLast: boolean, +): string[] { + const connector = isLast ? "└── " : "├── "; + const childPrefix = `${prefix}${isLast ? " " : "│ "}`; + const type = entry.event.span.type ? ` [${entry.event.span.type}]` : ""; + const lines = [`${prefix}${connector}${entry.name}${type}`]; + const fields = normalizeFields(entry.fields); + + for (const key of sortedFieldKeys(fields)) { + lines.push( + ...formatFieldBlock(fieldLabel(key), fields[key] as Json, childPrefix), + ); + } + + entry.children.forEach((child, index) => { + lines.push( + ...renderEntry(child, childPrefix, index === entry.children.length - 1), + ); + }); + + return lines; +} + +export function formatSpanTreeSnapshot( + entries: readonly (CapturedLogEvent | SpanTreeEntry)[], +): string { + const roots = buildEntries(entries); + return [ + "span_tree:", + ...roots.flatMap((entry, index) => + renderEntry(entry, "", index === roots.length - 1), + ), + "", + ].join("\n"); +} + +function jsonEntry(entry: NormalizedEntry): SpanTreeJsonNode { + const node: SpanTreeJsonNode = { + name: entry.name, + ...(entry.event.span.type ? { type: entry.event.span.type } : {}), + children: [], + }; + const fields = normalizeFields(entry.fields); + + for (const key of sortedFieldKeys(fields)) { + node[fieldLabel(key)] = fields[key] as Json; + } + + node.children = entry.children.map((child) => jsonEntry(child)); + return node; +} + +export function formatSpanTreeJsonSnapshot( + entries: readonly (CapturedLogEvent | SpanTreeEntry)[], +): string { + const roots = buildEntries(entries); + return `${JSON.stringify( + { + span_tree: roots.map((entry) => jsonEntry(entry)), + }, + null, + 2, + )}\n`; +} + +function txtSnapshotPath(jsonSnapshotPath: string): string { + return jsonSnapshotPath.endsWith(".json") + ? `${jsonSnapshotPath.slice(0, -".json".length)}.txt` + : `${jsonSnapshotPath}.txt`; +} + +export async function matchSpanTreeSnapshot( + entries: readonly (CapturedLogEvent | SpanTreeEntry)[], + jsonSnapshotPath: string, +): Promise { + await matchFileSnapshot( + formatSpanTreeSnapshot(entries), + txtSnapshotPath(jsonSnapshotPath), + ); + await matchFileSnapshot( + formatSpanTreeJsonSnapshot(entries), + jsonSnapshotPath, + ); +} diff --git a/e2e/helpers/trace-summary.ts b/e2e/helpers/trace-summary.ts index 09906921c..d31ea398c 100644 --- a/e2e/helpers/trace-summary.ts +++ b/e2e/helpers/trace-summary.ts @@ -74,9 +74,13 @@ function otlpAttributeValue(value: unknown): Json { if (typeof value.boolValue === "boolean") { return value.boolValue; } - if (typeof value.intValue === "string") { + if (typeof value.intValue === "number") { return value.intValue; } + if (typeof value.intValue === "string") { + const numericValue = Number(value.intValue); + return Number.isSafeInteger(numericValue) ? numericValue : value.intValue; + } if (typeof value.doubleValue === "number") { return value.doubleValue; } diff --git a/e2e/package.json b/e2e/package.json index 3bc0a27d5..d74de6d40 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -4,9 +4,10 @@ "private": true, "type": "module", "scripts": { - "test:e2e": "vitest run --run", - "test:e2e:canary": "node ./scripts/run-canary-tests-docker.mjs", - "test:e2e:record": "node ./scripts/run-record-tests.mjs" + "test:e2e": "node ./scripts/run-e2e-tests.mjs", + "test:e2e:update": "node ./scripts/run-e2e-tests.mjs --update", + "test:e2e:record": "node ./scripts/run-record-tests.mjs", + "test:e2e:canary": "node ./scripts/run-canary-tests-docker.mjs" }, "devDependencies": { "@braintrust/langchain-js": "workspace:^", diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v3.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v3.cassette.json index 93ed7248d..748007c95 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v3.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v3.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "66fa29eed4a03d8d", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:35:51.306Z", + "recordedAt": "2026-05-18T13:04:42.268Z", "request": { "body": { "kind": "json", @@ -41,12 +41,12 @@ } } ], - "created": 1778844951, - "id": "chatcmpl-Dfl7X423ZjmUcRVn7U5bPtjIXZSxE", + "created": 1779109482, + "id": "chatcmpl-DgrwAWP8cvz2oSUxcFx3v0Bwj2fBm", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_65d291db15", + "system_fingerprint": "fp_173518b8f7", "usage": { "completion_tokens": 2, "completion_tokens_details": { @@ -68,13 +68,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d76f3ffec2d4-VIE", + "cf-ray": "9fdb11b62ecd34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:35:51 GMT", + "date": "Mon, 18 May 2026 13:04:42 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "263", + "openai-processing-ms": "242", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -89,7 +89,7 @@ "x-ratelimit-remaining-tokens": "149999982", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_5ce18a4fc5764b5ab536e56654927178" + "x-request-id": "req_4c321266d6f04273a50d281c5f72c503" }, "status": 200, "statusText": "OK" @@ -99,7 +99,7 @@ "callIndex": 1, "id": "b9a0059ca6f02ace", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:35:51.980Z", + "recordedAt": "2026-05-18T13:04:42.826Z", "request": { "body": { "kind": "json", @@ -123,14 +123,14 @@ "response": { "body": { "chunks": [ - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"kaKWVl\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"One\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"OsVJx\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"C8nq0BK\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" two\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"fyxR\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"kg0PsqF\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" three\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"MR\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"TIff8AL\"}", - "data: {\"id\":\"chatcmpl-Dfl7XjxwCRG6rXmvaHVQQHL9AANR4\",\"object\":\"chat.completion.chunk\",\"created\":1778844951,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"bF\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Rl85Xj\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"One\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"vYxCV\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"K0BRVhb\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" two\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"zQLw\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Ohw996b\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" three\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"S7\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"WsLBLpe\"}", + "data: {\"id\":\"chatcmpl-DgrwAOLlrt0AEZ0YXxEQqa3RTNGu3\",\"object\":\"chat.completion.chunk\",\"created\":1779109482,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"tO\"}", "data: [DONE]" ], "kind": "sse" @@ -139,12 +139,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7725b62c2d4-VIE", + "cf-ray": "9fdb11b8bec134c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:35:51 GMT", + "date": "Mon, 18 May 2026 13:04:42 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "149", + "openai-processing-ms": "259", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -156,10 +156,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-remaining-tokens": "149999982", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_baed47d91a0f493db757e76d5537f0b6" + "x-request-id": "req_eca1f2bdfd12460cb51d24ff15639417" }, "status": 200, "statusText": "OK" @@ -169,7 +169,7 @@ "callIndex": 0, "id": "7d3c0bb6426e6be4", "matchKey": "POST api.openai.com/v1/embeddings", - "recordedAt": "2026-05-15T11:35:52.154Z", + "recordedAt": "2026-05-18T13:04:42.978Z", "request": { "body": { "kind": "json", @@ -757,13 +757,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d776998bc2d4-VIE", + "cf-ray": "9fdb11bc393b34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:35:52 GMT", + "date": "Mon, 18 May 2026 13:04:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "62", + "openai-processing-ms": "70", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -778,7 +778,7 @@ "x-ratelimit-remaining-tokens": "9999993", "x-ratelimit-reset-requests": "6ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_f0abdc22a9aa43a7a81298c1185bc2bc" + "x-request-id": "req_8b5cc0ce0ca94b3f80f14b4bc69fdd02" }, "status": 200, "statusText": "OK" @@ -788,7 +788,7 @@ "callIndex": 2, "id": "30f31184211ae8a1", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:35:52.924Z", + "recordedAt": "2026-05-18T13:04:43.761Z", "request": { "body": { "kind": "json", @@ -854,19 +854,19 @@ "arguments": "{\"location\":\"Paris, France\"}", "name": "get_weather" }, - "id": "call_mhv4d1s4zzniO9vzp8VQY0UP", + "id": "call_jyZMeyQdwYkaBnBPPl521T7X", "type": "function" } ] } } ], - "created": 1778844952, - "id": "chatcmpl-Dfl7YEbv5mpZEZIGwlEfo95rCsYq3", + "created": 1779109483, + "id": "chatcmpl-DgrwBwaiAlYUoi0y9JeFviOxqCeFS", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_af421e7311", + "system_fingerprint": "fp_9ee1f7f450", "usage": { "completion_tokens": 16, "completion_tokens_details": { @@ -888,13 +888,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7778b00c2d4-VIE", + "cf-ray": "9fdb11bd1bd634c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:35:52 GMT", + "date": "Mon, 18 May 2026 13:04:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "546", + "openai-processing-ms": "589", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -906,10 +906,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999950", + "x-ratelimit-remaining-tokens": "149999952", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_f666c030639945d088ec6bb24df167ca" + "x-request-id": "req_f06df73db9824b129e150f2386185d6d" }, "status": 200, "statusText": "OK" @@ -919,7 +919,7 @@ "callIndex": 3, "id": "9681fdcd86301e38", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:35:53.621Z", + "recordedAt": "2026-05-18T13:04:44.656Z", "request": { "body": { "kind": "json", @@ -985,19 +985,19 @@ "arguments": "{\"city\":\"Paris\"}", "name": "json" }, - "id": "call_t57L9tJjXinJ4Pgcf3w4RYXi", + "id": "call_OJCEpa2bhX7CnHdsNQ13hZwE", "type": "function" } ] } } ], - "created": 1778844953, - "id": "chatcmpl-Dfl7ZjuIW2f3Wr16JlFtCr49hfT8B", + "created": 1779109484, + "id": "chatcmpl-DgrwCwQPKL1OiQczwJQzQrPyFf2bI", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_5b8b54db47", + "system_fingerprint": "fp_54f26dc974", "usage": { "completion_tokens": 5, "completion_tokens_details": { @@ -1019,13 +1019,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d77c58a1c2d4-VIE", + "cf-ray": "9fdb11c20b7534c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:35:53 GMT", + "date": "Mon, 18 May 2026 13:04:44 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "434", + "openai-processing-ms": "351", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1040,7 +1040,7 @@ "x-ratelimit-remaining-tokens": "149999990", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_509ed2ecc015413ab0bfda835e350469" + "x-request-id": "req_ffe0812c829941d8ad81d3708fc39b0c" }, "status": 200, "statusText": "OK" @@ -1050,7 +1050,7 @@ "callIndex": 4, "id": "d2e0c4c7403336d9", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:35:54.054Z", + "recordedAt": "2026-05-18T13:04:45.140Z", "request": { "body": { "kind": "json", @@ -1100,13 +1100,13 @@ "response": { "body": { "chunks": [ - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_y52ztt8SCJJtMEZAuOK42QYh\",\"type\":\"function\",\"function\":{\"name\":\"json\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"XawPWTs8\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"TNU5FB9OxFZ\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"rcC3fIiU4Y\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"sciqGdTp5\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"4GNn1iUXk\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"oVncRHr3Dsa\"}", - "data: {\"id\":\"chatcmpl-Dfl7Z4JjWClOFzGd7nsnfAPE14PuL\",\"object\":\"chat.completion.chunk\",\"created\":1778844953,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"6Q\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_4KgGzMWOq6YujSPTokSDfUzn\",\"type\":\"function\",\"function\":{\"name\":\"json\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"DqZoNsxI\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"ZNpe6lobrrk\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"5ElcQy0YMh\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"JqsOZk759\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"EWpSOqdGD\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"IPF5sbxqJOc\"}", + "data: {\"id\":\"chatcmpl-DgrwCm90CiwZDAnHJRFnisoytubfF\",\"object\":\"chat.completion.chunk\",\"created\":1779109484,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"MD\"}", "data: [DONE]" ], "kind": "sse" @@ -1115,12 +1115,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d780bd4cc2d4-VIE", + "cf-ray": "9fdb11c79d5534c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:35:54 GMT", + "date": "Mon, 18 May 2026 13:04:45 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "146", + "openai-processing-ms": "219", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1135,7 +1135,7 @@ "x-ratelimit-remaining-tokens": "149999990", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_b64a9bac0fec47a0ab1bf251a03eb0cd" + "x-request-id": "req_9ab89e637d564ed2ada6d93979c6e35a" }, "status": 200, "statusText": "OK" diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v4.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v4.cassette.json index 88ba44739..5eef69872 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v4.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v4.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "66fa29eed4a03d8d", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:06.831Z", + "recordedAt": "2026-05-18T13:04:52.052Z", "request": { "body": { "kind": "json", @@ -41,12 +41,12 @@ } } ], - "created": 1778844966, - "id": "chatcmpl-Dfl7mVj9J3KYiqw5iflcuqgRvvXjy", + "created": 1779109491, + "id": "chatcmpl-DgrwJ957VcnuGWpD4L62d05UcKDGe", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_65d291db15", + "system_fingerprint": "fp_173518b8f7", "usage": { "completion_tokens": 2, "completion_tokens_details": { @@ -68,13 +68,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7d03a94c2d4-VIE", + "cf-ray": "9fdb11f2c81d34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:36:06 GMT", + "date": "Mon, 18 May 2026 13:04:52 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "280", + "openai-processing-ms": "298", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -86,10 +86,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999982", + "x-ratelimit-remaining-tokens": "149999985", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_559754a013c74307994c05d714589366" + "x-request-id": "req_604b6dbfc4b045379b506719f5156289" }, "status": 200, "statusText": "OK" @@ -99,7 +99,7 @@ "callIndex": 1, "id": "b9a0059ca6f02ace", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:07.472Z", + "recordedAt": "2026-05-18T13:04:52.988Z", "request": { "body": { "kind": "json", @@ -143,12 +143,12 @@ } } ], - "created": 1778844967, - "id": "chatcmpl-Dfl7nMkrQxvlPVKuGob2rpU0BNj0n", + "created": 1779109492, + "id": "chatcmpl-DgrwK4obu6zo5ZDuEC5QHrFsTlh5h", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_09be0b3c41", + "system_fingerprint": "fp_0ed05801bb", "usage": { "completion_tokens": 5, "completion_tokens_details": { @@ -170,13 +170,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7d35de9c2d4-VIE", + "cf-ray": "9fdb11f5ebd834c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:36:07 GMT", + "date": "Mon, 18 May 2026 13:04:53 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "374", + "openai-processing-ms": "730", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -188,10 +188,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999920", + "x-ratelimit-remaining-tokens": "149999922", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_96a090c33e1047359aae13a266e66059" + "x-request-id": "req_6575c998cd284486a061bc4e3520fbeb" }, "status": 200, "statusText": "OK" @@ -201,7 +201,7 @@ "callIndex": 2, "id": "30f31184211ae8a1", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:08.022Z", + "recordedAt": "2026-05-18T13:04:53.617Z", "request": { "body": { "kind": "json", @@ -225,14 +225,14 @@ "response": { "body": { "chunks": [ - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"LbMepT\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"One\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"uVXqF\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"fv6DdSe\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" two\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"PtCS\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"sDmSosq\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" three\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"n3\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"7CgCfE5\"}", - "data: {\"id\":\"chatcmpl-Dfl7nW2pp5O2ZAtu5dvCP0zzQpp8T\",\"object\":\"chat.completion.chunk\",\"created\":1778844967,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_ac7a1e4bb7\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"Ve\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"nWKqZq\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"One\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Zj0TR\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"aGzsbiD\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" two\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"oC0F\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"VcoOd2H\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" three\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"L1\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"8XIj2wD\"}", + "data: {\"id\":\"chatcmpl-DgrwLrVp6Z9Ijz259L41P1O6v1x9w\",\"object\":\"chat.completion.chunk\",\"created\":1779109493,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"i7\"}", "data: [DONE]" ], "kind": "sse" @@ -241,12 +241,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7d75a35c2d4-VIE", + "cf-ray": "9fdb11fbae9e34c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:36:07 GMT", + "date": "Mon, 18 May 2026 13:04:53 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "179", + "openai-processing-ms": "236", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -261,7 +261,7 @@ "x-ratelimit-remaining-tokens": "149999985", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_25708029a59e46dd8ef7e9bbae7cb21b" + "x-request-id": "req_def116eb67c74ecf8a1621dd74dc426a" }, "status": 200, "statusText": "OK" @@ -271,7 +271,7 @@ "callIndex": 0, "id": "7d3c0bb6426e6be4", "matchKey": "POST api.openai.com/v1/embeddings", - "recordedAt": "2026-05-15T11:36:08.183Z", + "recordedAt": "2026-05-18T13:04:53.719Z", "request": { "body": { "kind": "json", @@ -292,556 +292,559 @@ "data": [ { "embedding": [ - 0.0196075439453125, 0.019073486328125, -0.01314544677734375, - -0.020660400390625, -0.0188140869140625, 0.029632568359375, - -0.0206298828125, 0.0236358642578125, 0.006320953369140625, - -0.01708984375, 0.004039764404296875, 0.0103607177734375, - -0.0289459228515625, 0.0068359375, -0.03192138671875, - 0.0251922607421875, -0.0282745361328125, 0.050262451171875, - 0.0914306640625, -0.02008056640625, 0.0222930908203125, - -0.01297760009765625, -0.0144805908203125, 0.034393310546875, - 0.052734375, -0.0011014938354492188, 0.00817108154296875, - 0.042510986328125, 0.0030460357666015625, 0.05987548828125, - 0.0164642333984375, -0.00787353515625, -0.00244903564453125, - 0.02423095703125, 0.01953125, -0.061065673828125, - -0.013336181640625, -0.03851318359375, 0.0079498291015625, - 0.0276641845703125, 0.0171051025390625, -0.02838134765625, - -0.0286865234375, 0.03277587890625, -0.035919189453125, - -0.047088623046875, -0.00940704345703125, -0.0224761962890625, - 0.02313232421875, -0.0037555694580078125, 0.045654296875, - -0.004848480224609375, 0.02020263671875, 0.00923919677734375, - 0.0044708251953125, 0.02252197265625, 0.03271484375, - 0.0433349609375, 0.0257110595703125, 0.0035400390625, - 0.051788330078125, 0.0206298828125, 0.04046630859375, - 0.01776123046875, -0.00559234619140625, 0.03271484375, - 0.021575927734375, 0.0322265625, 0.050048828125, - 0.02581787109375, 0.0187835693359375, 0.044342041015625, - -0.005947113037109375, 0.006748199462890625, 0.01885986328125, - 0.014678955078125, 0.004638671875, 0.038848876953125, - 0.0173492431640625, -0.03466796875, -0.0016269683837890625, - 0.01055908203125, 0.01372528076171875, -0.02166748046875, - -0.00677490234375, -0.01284027099609375, -0.00388336181640625, - 0.00274658203125, -0.002719879150390625, -0.0220794677734375, - 0.0014619827270507812, -0.01175689697265625, -0.0172119140625, - 0.015655517578125, -0.0308990478515625, 0.00458526611328125, - 0.0211334228515625, -0.04888916015625, -0.058837890625, - 0.02716064453125, 0.027862548828125, 0.001308441162109375, - -0.04681396484375, 0.0316162109375, 0.0155029296875, - -0.01544952392578125, 0.0004296302795410156, - -0.0005507469177246094, 0.01428985595703125, -0.0408935546875, - -0.021759033203125, 0.00412750244140625, -0.00025177001953125, - 0.032440185546875, -0.059783935546875, 0.01093292236328125, - 0.01397705078125, -0.00713348388671875, 0.031005859375, - -0.007572174072265625, -0.006343841552734375, - 0.0026073455810546875, -0.04180908203125, 0.040374755859375, - -0.03387451171875, -0.048248291015625, 0.006519317626953125, - 0.0264739990234375, 0.0227203369140625, -0.00856781005859375, - 0.004657745361328125, -0.044952392578125, 0.016876220703125, - 0.0258941650390625, 0.0506591796875, -0.00958251953125, - 0.033721923828125, -0.0057525634765625, 0.01751708984375, - 0.0177154541015625, -0.01558685302734375, 0.028533935546875, - -0.01058197021484375, 0.036529541015625, -0.0183258056640625, - -0.033843994140625, -0.031402587890625, 0.040069580078125, - 0.01015472412109375, -0.04278564453125, 0.016510009765625, - -0.0212554931640625, 0.00275421142578125, 0.0289459228515625, - -0.0037078857421875, -0.062103271484375, 0.01442718505859375, - -0.0018835067749023438, 0.0384521484375, 0.02569580078125, - -0.036865234375, 0.021331787109375, -0.046173095703125, - -0.016265869140625, -0.0034809112548828125, - 0.0081939697265625, -0.0090179443359375, -0.00542449951171875, - 0.011871337890625, 0.0159454345703125, -0.0235137939453125, - 0.00461578369140625, 0.051116943359375, 0.0389404296875, - 0.0225677490234375, -0.01392364501953125, -0.0185699462890625, - 0.08013916015625, 0.0262603759765625, -0.02142333984375, - -0.01103973388671875, -0.01428985595703125, 0.030731201171875, - -0.01473236083984375, 0.0189361572265625, 0.0250244140625, - -0.0206451416015625, -0.026885986328125, -0.0172882080078125, - 0.05322265625, 0.0615234375, -0.019073486328125, - 0.056396484375, -0.045440673828125, -0.06280517578125, - 0.0241546630859375, 0.0021800994873046875, - 0.006801605224609375, 0.0093231201171875, 0.00798797607421875, - -0.03924560546875, 0.04193115234375, -0.0056304931640625, - -0.00327301025390625, -0.01485443115234375, - 0.0191192626953125, -0.01004791259765625, -0.048309326171875, - 0.0295257568359375, -0.004802703857421875, - -0.0124359130859375, -0.00547027587890625, - -0.0262298583984375, 0.0384521484375, 0.03857421875, - -0.01038360595703125, 0.012908935546875, 0.034027099609375, - 0.0299072265625, -0.00550079345703125, 0.034637451171875, - 0.023406982421875, 0.0113067626953125, -0.01557159423828125, - -0.048919677734375, 0.055419921875, 0.052642822265625, - 0.00609588623046875, -0.044586181640625, 0.002712249755859375, - 0.00432586669921875, -0.028656005859375, -0.0147552490234375, - -0.015411376953125, -0.0269775390625, -0.01922607421875, - 0.003505706787109375, -0.0132598876953125, -0.04779052734375, - 0.019317626953125, -0.0338134765625, 0.01849365234375, - 0.01416015625, 0.0026702880859375, -0.00037384033203125, - 0.0026149749755859375, 0.01383209228515625, 0.037200927734375, - -0.0421142578125, 0.0126953125, 0.043365478515625, - -0.0171966552734375, -0.025360107421875, 0.017303466796875, - -0.03546142578125, 0.035125732421875, 0.006137847900390625, - -0.00630950927734375, -0.047698974609375, -0.0099029541015625, - 0.01043701171875, 0.0198974609375, -0.00995635986328125, - -0.0018243789672851562, -0.00839996337890625, - 0.0192108154296875, -0.0095977783203125, -0.023651123046875, - -0.0224609375, -0.030517578125, 0.03485107421875, - -0.0008287429809570312, -0.0177154541015625, 0.029052734375, - -0.020965576171875, 0.005268096923828125, -0.028167724609375, - 0.0009093284606933594, -0.001140594482421875, - 0.0157928466796875, -0.02178955078125, 0.0443115234375, - -0.01227569580078125, 0.01256561279296875, - -0.0095367431640625, -0.0198211669921875, 0.04046630859375, - 0.06732177734375, -0.032562255859375, 0.052032470703125, - 0.02166748046875, -0.021209716796875, -0.033935546875, - 0.0225982666015625, -0.008056640625, 0.006862640380859375, - 0.035552978515625, 0.0010929107666015625, - 0.006832122802734375, 0.012481689453125, -0.0239105224609375, - -0.0117950439453125, 0.004608154296875, 0.01525115966796875, - -0.049560546875, 0.048431396484375, -0.045501708984375, - 0.0196685791015625, 0.0521240234375, 0.00585174560546875, - -0.054107666015625, -0.036376953125, 0.04632568359375, - -0.06768798828125, -0.01151275634765625, 0.00713348388671875, - -0.0157928466796875, 0.0195770263671875, 0.03448486328125, - 0.01102447509765625, -0.00614166259765625, - 0.01256561279296875, 0.007549285888671875, - 0.0020580291748046875, -0.0272674560546875, 0.031707763671875, - 0.0144805908203125, 0.03839111328125, -0.0278778076171875, - -0.00634765625, 0.0008325576782226562, 0.00878143310546875, - -0.008514404296875, 0.06134033203125, -0.040191650390625, - -0.021484375, 0.003170013427734375, 0.029327392578125, - 0.003055572509765625, -0.00859832763671875, -0.02532958984375, - 0.0270538330078125, -0.035919189453125, 0.0270233154296875, - -0.0033893585205078125, -0.01428985595703125, 0.0377197265625, - 0.001129150390625, -0.005687713623046875, -0.03594970703125, - -0.0191802978515625, -0.0195770263671875, - 0.0004911422729492188, 0.0313720703125, -0.040771484375, - 0.0191802978515625, 0.06414794921875, 0.00946807861328125, - -0.05694580078125, 0.0130462646484375, -0.016510009765625, - -0.052764892578125, -0.0163116455078125, -0.018951416015625, - -0.03668212890625, -0.0084381103515625, - -0.0017042160034179688, -0.00557708740234375, + 0.0196380615234375, 0.019073486328125, -0.01320648193359375, + -0.0206451416015625, -0.0188751220703125, 0.02972412109375, + -0.020599365234375, 0.023651123046875, 0.006359100341796875, + -0.017059326171875, 0.004039764404296875, 0.01035308837890625, + -0.028961181640625, 0.00681304931640625, -0.03192138671875, + 0.02520751953125, -0.0282440185546875, 0.0501708984375, + 0.09149169921875, -0.020050048828125, 0.022308349609375, + -0.01296234130859375, -0.01445770263671875, 0.03436279296875, + 0.052764892578125, -0.0010938644409179688, 0.0081634521484375, + 0.042510986328125, 0.003047943115234375, 0.059844970703125, + 0.0164337158203125, -0.0078582763671875, + -0.0024394989013671875, 0.02423095703125, 0.0195159912109375, + -0.061065673828125, -0.0133514404296875, -0.038421630859375, + 0.00791168212890625, 0.027679443359375, 0.01708984375, + -0.0284271240234375, -0.0287322998046875, 0.03277587890625, + -0.035919189453125, -0.04705810546875, -0.0094451904296875, + -0.022491455078125, 0.02313232421875, -0.0037364959716796875, + 0.045654296875, -0.00487518310546875, 0.020172119140625, + 0.009246826171875, 0.00449371337890625, 0.022552490234375, + 0.03277587890625, 0.0433349609375, 0.02569580078125, + 0.0035457611083984375, 0.051788330078125, 0.020599365234375, + 0.04046630859375, 0.0177459716796875, -0.0055999755859375, + 0.03271484375, 0.0215606689453125, 0.032257080078125, + 0.050048828125, 0.0258026123046875, 0.018768310546875, + 0.044342041015625, -0.005954742431640625, 0.00675201416015625, + 0.0188751220703125, 0.01470947265625, 0.00463104248046875, + 0.038787841796875, 0.017364501953125, -0.03466796875, + -0.0016241073608398438, 0.01056671142578125, + 0.0137481689453125, -0.0216827392578125, -0.00678253173828125, + -0.01279449462890625, -0.00383758544921875, + 0.0027370452880859375, -0.0027408599853515625, + -0.0220947265625, 0.0014705657958984375, -0.0117034912109375, + -0.0172882080078125, 0.015655517578125, -0.03094482421875, + 0.00457000732421875, 0.0211029052734375, -0.04888916015625, + -0.058837890625, 0.02716064453125, 0.0279083251953125, + 0.0013456344604492188, -0.046783447265625, 0.0316162109375, + 0.0154876708984375, -0.01546478271484375, + 0.00037980079650878906, -0.0005192756652832031, + 0.0143280029296875, -0.040924072265625, -0.021728515625, + 0.004119873046875, -0.0002334117889404297, 0.032440185546875, + -0.059783935546875, 0.01092529296875, 0.014007568359375, + -0.00711822509765625, 0.031036376953125, + -0.007556915283203125, -0.00634002685546875, 0.0025634765625, + -0.04180908203125, 0.0404052734375, -0.03387451171875, + -0.048248291015625, 0.00656890869140625, 0.0265350341796875, + 0.0227813720703125, -0.00855255126953125, 0.00469970703125, + -0.044952392578125, 0.01690673828125, 0.025909423828125, + 0.05059814453125, -0.0096282958984375, 0.033782958984375, + -0.00574493408203125, 0.0175018310546875, 0.01776123046875, + -0.015625, 0.0285797119140625, -0.01055908203125, + 0.0364990234375, -0.018280029296875, -0.0338134765625, + -0.0313720703125, 0.040069580078125, 0.0101470947265625, + -0.04278564453125, 0.0164794921875, -0.0212554931640625, + 0.00272369384765625, 0.0289459228515625, + -0.003696441650390625, -0.0621337890625, 0.014434814453125, + -0.00196075439453125, 0.0384521484375, 0.025665283203125, + -0.036865234375, 0.0212860107421875, -0.046142578125, + -0.0162811279296875, -0.0034656524658203125, + 0.00821685791015625, -0.00901031494140625, + -0.00539398193359375, 0.011871337890625, 0.0160064697265625, + -0.0234527587890625, 0.004573822021484375, 0.0511474609375, + 0.038909912109375, 0.0225677490234375, -0.0139312744140625, + -0.018524169921875, 0.08013916015625, 0.026275634765625, + -0.0214385986328125, -0.01105499267578125, + -0.01430511474609375, 0.03076171875, -0.01468658447265625, + 0.0189056396484375, 0.0250244140625, -0.0206146240234375, + -0.0269317626953125, -0.0172882080078125, 0.05322265625, + 0.061492919921875, -0.0190887451171875, 0.056365966796875, + -0.04541015625, -0.0628662109375, 0.0241546630859375, + 0.002185821533203125, 0.0067596435546875, 0.00933074951171875, + 0.0079803466796875, -0.039215087890625, 0.041900634765625, + -0.0055999755859375, -0.0033016204833984375, + -0.01486968994140625, 0.019073486328125, -0.0100555419921875, + -0.04833984375, 0.029510498046875, -0.00475311279296875, + -0.01241302490234375, -0.005481719970703125, + -0.0262298583984375, 0.038421630859375, 0.038543701171875, + -0.01038360595703125, 0.012939453125, 0.033966064453125, + 0.0299072265625, -0.005496978759765625, 0.03466796875, + 0.023406982421875, 0.0113525390625, -0.0155792236328125, + -0.048919677734375, 0.055419921875, 0.05267333984375, + 0.00608062744140625, -0.04461669921875, 0.002704620361328125, + 0.00434112548828125, -0.0286407470703125, + -0.01476287841796875, -0.01540374755859375, + -0.0269317626953125, -0.019195556640625, 0.0035400390625, + -0.01326751708984375, -0.047821044921875, 0.0193634033203125, + -0.033843994140625, 0.0185394287109375, 0.0141448974609375, + 0.0026836395263671875, -0.00033974647521972656, + 0.002651214599609375, 0.01383209228515625, 0.03717041015625, + -0.042083740234375, 0.01270294189453125, 0.04339599609375, + -0.0172119140625, -0.025360107421875, 0.0172576904296875, + -0.035430908203125, 0.035125732421875, 0.00611114501953125, + -0.00634002685546875, -0.047698974609375, + -0.00983428955078125, 0.0104217529296875, 0.019866943359375, + -0.00997161865234375, -0.0018606185913085938, + -0.008392333984375, 0.019195556640625, -0.00960540771484375, + -0.023651123046875, -0.022491455078125, -0.030517578125, + 0.034912109375, -0.0008358955383300781, -0.0177154541015625, + 0.0290985107421875, -0.020965576171875, 0.005283355712890625, + -0.028228759765625, 0.0008840560913085938, -0.001129150390625, + 0.015777587890625, -0.0218505859375, 0.0443115234375, + -0.01230621337890625, 0.01255035400390625, + -0.00955963134765625, -0.01983642578125, 0.04046630859375, + 0.0672607421875, -0.0325927734375, 0.052032470703125, + 0.02166748046875, -0.021240234375, -0.033966064453125, + 0.022613525390625, -0.008087158203125, 0.006885528564453125, + 0.03558349609375, 0.0010814666748046875, 0.006816864013671875, + 0.0125274658203125, -0.0239105224609375, -0.01180267333984375, + 0.004608154296875, 0.0152740478515625, -0.04949951171875, + 0.0484619140625, -0.045501708984375, 0.019683837890625, + 0.0521240234375, 0.005802154541015625, -0.054107666015625, + -0.03643798828125, 0.04638671875, -0.0677490234375, + -0.01151275634765625, 0.007213592529296875, + -0.0158538818359375, 0.019561767578125, 0.03448486328125, + 0.011016845703125, -0.0061187744140625, 0.0125732421875, + 0.007537841796875, 0.002048492431640625, -0.0272369384765625, + 0.03173828125, 0.01447296142578125, 0.03839111328125, + -0.02783203125, -0.0063323974609375, 0.0008225440979003906, + 0.00875091552734375, -0.00853729248046875, 0.06134033203125, + -0.0401611328125, -0.021484375, 0.0031414031982421875, + 0.029296875, 0.003040313720703125, -0.0085601806640625, + -0.0253448486328125, 0.0270233154296875, -0.035919189453125, + 0.0270233154296875, -0.0033855438232421875, + -0.01428985595703125, 0.0377197265625, 0.0011196136474609375, + -0.00563812255859375, -0.03594970703125, -0.0191650390625, + -0.0196075439453125, 0.0004887580871582031, 0.0313720703125, + -0.040771484375, 0.0192108154296875, 0.064208984375, + 0.0094451904296875, -0.05694580078125, 0.01303863525390625, + -0.0164947509765625, -0.052764892578125, -0.0163116455078125, + -0.018951416015625, -0.036712646484375, -0.0084228515625, + -0.0016994476318359375, -0.005573272705078125, -0.03851318359375, 0.07061767578125, -0.049896240234375, - -0.0277252197265625, 0.0287322998046875, -0.02880859375, - 0.047088623046875, -0.030364990234375, -0.025054931640625, - -0.008270263671875, -0.04766845703125, -0.02618408203125, - -0.0643310546875, -0.002410888671875, 0.0355224609375, - -0.04705810546875, 0.03204345703125, 0.0369873046875, - 0.043701171875, -0.0550537109375, 0.01226043701171875, - -0.006168365478515625, 0.012237548828125, - 0.004772186279296875, 0.00537872314453125, 0.022552490234375, - -0.03302001953125, -0.0009675025939941406, -0.0103759765625, - -0.00447845458984375, -0.029510498046875, -0.0172576904296875, - 0.0028629302978515625, -0.0390625, 0.01015472412109375, - -0.006866455078125, 0.04296875, 0.028167724609375, - 0.0075225830078125, -0.01715087890625, 0.0215911865234375, - -0.0222625732421875, 0.006572723388671875, -0.037872314453125, - 0.015380859375, -0.003780364990234375, 0.0131683349609375, - 0.0540771484375, 0.0298919677734375, -0.07373046875, - 0.0330810546875, -0.02154541015625, 0.037506103515625, - 0.004802703857421875, -0.058837890625, 0.0113525390625, - 0.0028858184814453125, -0.007354736328125, - -0.01139068603515625, -0.06524658203125, -0.019317626953125, - 0.032623291015625, 0.00508880615234375, 0.021331787109375, - 0.0006260871887207031, -0.008514404296875, - 0.01102447509765625, 0.0150909423828125, 0.0099639892578125, - 0.01019287109375, -0.01428985595703125, 0.00167083740234375, - 0.047515869140625, -0.017364501953125, 0.01197052001953125, - -0.03564453125, -0.0462646484375, -0.017425537109375, - 0.055816650390625, 0.029937744140625, -0.045867919921875, - 0.06005859375, -0.0311737060546875, -0.00281524658203125, - -0.00878143310546875, -0.01318359375, -0.0246124267578125, - 0.0300140380859375, 0.0197296142578125, 0.0699462890625, - 0.00809478759765625, -0.03424072265625, 0.00490570068359375, - 0.046600341796875, -0.00392913818359375, 0.044769287109375, - 0.03271484375, 0.018768310546875, 0.0243072509765625, - 0.019378662109375, -0.038787841796875, -0.03131103515625, - 0.0158538818359375, 0.0335693359375, 0.0224456787109375, - 0.025848388671875, 0.007061004638671875, 0.0049285888671875, - -0.0595703125, 0.038726806640625, -0.0494384765625, - -0.06390380859375, -0.0011005401611328125, 0.0219268798828125, - -0.018798828125, 0.006317138671875, -0.0794677734375, - -0.00662994384765625, 0.00353240966796875, - 0.0020122528076171875, -0.0234527587890625, - 0.0159759521484375, 0.052032470703125, -0.038360595703125, - 0.01372528076171875, -0.00020372867584228516, - 0.021575927734375, -0.00679779052734375, -0.053985595703125, - -0.01056671142578125, -0.0272369384765625, - -0.0236053466796875, 0.023162841796875, 0.00165557861328125, - 0.0499267578125, 0.012908935546875, 0.033233642578125, - 0.08331298828125, 0.01042938232421875, 0.0039825439453125, - 0.00824737548828125, -0.0260772705078125, 0.0299530029296875, - -0.025299072265625, 0.032196044921875, 0.01166534423828125, - -0.010711669921875, -0.00376129150390625, -0.023162841796875, - 0.0157470703125, 0.01163482666015625, 0.002651214599609375, - 0.0198822021484375, 0.0246124267578125, -0.0152740478515625, - 0.004291534423828125, 0.032928466796875, 0.0141143798828125, - -0.024322509765625, -0.007518768310546875, 0.04766845703125, - -0.04901123046875, 0.0227508544921875, -0.0255584716796875, - -0.0141143798828125, -0.01641845703125, 0.0095367431640625, - 0.0183563232421875, 0.0184173583984375, 0.05340576171875, - 0.00008529424667358398, 0.03070068359375, - 0.0038967132568359375, -0.0161285400390625, - -0.006809234619140625, -0.0189361572265625, - -0.01512908935546875, -0.0167999267578125, 0.022003173828125, - -0.027374267578125, -0.03448486328125, -0.016998291015625, + -0.027740478515625, 0.0287628173828125, -0.028839111328125, + 0.04705810546875, -0.0303955078125, -0.0250396728515625, + -0.008270263671875, -0.047637939453125, -0.0262298583984375, + -0.0643310546875, -0.0024242401123046875, 0.0355224609375, + -0.047149658203125, 0.032012939453125, 0.03704833984375, + 0.043701171875, -0.05511474609375, 0.01226806640625, + -0.006153106689453125, 0.012237548828125, 0.00476837158203125, + 0.00540924072265625, 0.02252197265625, -0.03302001953125, + -0.0009093284606933594, -0.0104217529296875, + -0.00444793701171875, -0.0295257568359375, + -0.0172882080078125, 0.002864837646484375, -0.039031982421875, + 0.01010894775390625, -0.006862640380859375, 0.042938232421875, + 0.028167724609375, 0.007511138916015625, -0.0171661376953125, + 0.0216217041015625, -0.022247314453125, 0.00659942626953125, + -0.03790283203125, 0.01537322998046875, + -0.0037822723388671875, 0.01319122314453125, + 0.054046630859375, 0.0298919677734375, -0.07373046875, + 0.033111572265625, -0.021514892578125, 0.03753662109375, + 0.004749298095703125, -0.058837890625, 0.01134490966796875, + 0.0028514862060546875, -0.007328033447265625, + -0.01137542724609375, -0.06524658203125, -0.0193023681640625, + 0.03265380859375, 0.00508880615234375, 0.0213623046875, + 0.0005936622619628906, -0.0084991455078125, + 0.0110015869140625, 0.01507568359375, 0.0099639892578125, + 0.01019287109375, -0.01430511474609375, 0.0016298294067382812, + 0.047576904296875, -0.017364501953125, 0.01197052001953125, + -0.03570556640625, -0.0462646484375, -0.0174102783203125, + 0.055755615234375, 0.0299072265625, -0.045867919921875, + 0.060028076171875, -0.031219482421875, -0.002826690673828125, + -0.0088043212890625, -0.01319122314453125, + -0.0246429443359375, 0.0300750732421875, 0.0197296142578125, + 0.06988525390625, 0.00811767578125, -0.03424072265625, + 0.004909515380859375, 0.046600341796875, + -0.0038928985595703125, 0.044769287109375, 0.032745361328125, + 0.018707275390625, 0.024261474609375, 0.0193939208984375, + -0.038818359375, -0.031280517578125, 0.015869140625, + 0.0335693359375, 0.0224456787109375, 0.02581787109375, + 0.007049560546875, 0.00487518310546875, -0.059600830078125, + 0.03875732421875, -0.0494384765625, -0.06390380859375, + -0.0011310577392578125, 0.021942138671875, + -0.0187530517578125, 0.0063018798828125, -0.0794677734375, + -0.00656890869140625, 0.00356292724609375, + 0.0019741058349609375, -0.0234832763671875, + 0.0159759521484375, 0.052032470703125, -0.038421630859375, + 0.01377105712890625, -0.00019228458404541016, + 0.0215606689453125, -0.006824493408203125, -0.053955078125, + -0.0105743408203125, -0.0272216796875, -0.02362060546875, + 0.023193359375, 0.0016756057739257812, 0.0499267578125, + 0.0129547119140625, 0.033172607421875, 0.08331298828125, + 0.0103759765625, 0.003932952880859375, 0.00826263427734375, + -0.0260772705078125, 0.0300445556640625, -0.0252685546875, + 0.0322265625, 0.01165008544921875, -0.01071929931640625, + -0.0037631988525390625, -0.02313232421875, 0.0157470703125, + 0.0116424560546875, 0.0026683807373046875, 0.0199127197265625, + 0.0246124267578125, -0.01528167724609375, 0.004302978515625, + 0.032928466796875, 0.01409149169921875, -0.024322509765625, + -0.007480621337890625, 0.04766845703125, -0.04901123046875, + 0.0227203369140625, -0.0255889892578125, -0.014129638671875, + -0.0164337158203125, 0.0095062255859375, 0.01837158203125, + 0.01837158203125, 0.05340576171875, 0.00007402896881103516, + 0.03070068359375, 0.003925323486328125, -0.0161285400390625, + -0.00675201416015625, -0.0189666748046875, + -0.01511383056640625, -0.0167999267578125, 0.0219573974609375, + -0.0273590087890625, -0.034515380859375, -0.01702880859375, -0.024627685546875, 0.045806884765625, -0.03558349609375, - -0.0439453125, -0.000728607177734375, -0.0036830902099609375, - -0.009368896484375, -0.0039520263671875, 0.026947021484375, - -0.01522064208984375, -0.00968170166015625, 0.06329345703125, - 0.037994384765625, 0.017242431640625, 0.0188751220703125, - 0.036865234375, 0.00426483154296875, 0.01039886474609375, - -0.01483154296875, -0.022216796875, -0.0118408203125, - 0.0484619140625, 0.01296234130859375, 0.0175018310546875, - 0.02081298828125, 0.0182037353515625, 0.010162353515625, - -0.003997802734375, -0.0299224853515625, 0.0654296875, - -0.0054168701171875, 0.01355743408203125, 0.032470703125, - -0.03521728515625, -0.04522705078125, 0.0218353271484375, - 0.0035457611083984375, -0.002376556396484375, -0.0419921875, - -0.0294342041015625, -0.0037860870361328125, - 0.01380157470703125, 0.0174713134765625, -0.0234527587890625, - 0.01428985595703125, 0.0014009475708007812, 0.01141357421875, - 0.0008606910705566406, -0.035858154296875, - -0.01201629638671875, 0.00235748291015625, 0.0251312255859375, - -0.019287109375, -0.0242156982421875, 0.01537322998046875, - -0.036956787109375, 0.00670623779296875, -0.0191650390625, - 0.0223846435546875, -0.007534027099609375, 0.0176849365234375, - -0.0187530517578125, 0.01541900634765625, 0.028289794921875, - -0.005496978759765625, 0.02801513671875, -0.02410888671875, - 0.023956298828125, -0.01284027099609375, -0.00730133056640625, - 0.01430511474609375, 0.03460693359375, -0.031982421875, - 0.0055694580078125, -0.005382537841796875, 0.047088623046875, - 0.0189361572265625, 0.0183868408203125, -0.0099945068359375, - -0.0172576904296875, 0.024444580078125, 0.0141754150390625, - 0.045562744140625, 0.0128021240234375, -0.0233917236328125, - -0.0251007080078125, -0.004085540771484375, - -0.0197906494140625, 0.008392333984375, -0.03021240234375, - 0.041046142578125, 0.0124664306640625, -0.0015897750854492188, - 0.0059967041015625, -0.006702423095703125, 0.0124664306640625, + -0.0439453125, -0.0007295608520507812, -0.0037212371826171875, + -0.00934600830078125, -0.00396728515625, 0.0269317626953125, + -0.01520538330078125, -0.00968170166015625, 0.063232421875, + 0.0379638671875, 0.0172882080078125, 0.01885986328125, + 0.036865234375, 0.00424957275390625, 0.010406494140625, + -0.01483917236328125, -0.022186279296875, + -0.01180267333984375, 0.0484619140625, 0.01297760009765625, + 0.0174713134765625, 0.020751953125, 0.01824951171875, + 0.01013946533203125, -0.00399017333984375, -0.029998779296875, + 0.0654296875, -0.005413055419921875, 0.0135650634765625, + 0.032470703125, -0.03521728515625, -0.04522705078125, + 0.0218505859375, 0.003498077392578125, -0.002376556396484375, + -0.0419921875, -0.0294647216796875, -0.003753662109375, + 0.01380157470703125, 0.0174560546875, -0.0234375, + 0.01435089111328125, 0.0014057159423828125, + 0.0113372802734375, 0.000843048095703125, -0.035858154296875, + -0.01200103759765625, 0.0024013519287109375, + 0.0251007080078125, -0.0193023681640625, -0.0242462158203125, + 0.01535797119140625, -0.0369873046875, 0.00672149658203125, + -0.0191650390625, 0.022369384765625, -0.0075225830078125, + 0.0176544189453125, -0.0187530517578125, 0.0154266357421875, + 0.028350830078125, -0.00547027587890625, 0.0279541015625, + -0.0240631103515625, 0.02398681640625, -0.0128021240234375, + -0.007293701171875, 0.0142974853515625, 0.03460693359375, + -0.032012939453125, 0.005584716796875, -0.00534820556640625, + 0.04705810546875, 0.018890380859375, 0.0184326171875, + -0.01000213623046875, -0.0172882080078125, 0.0244140625, + 0.01416778564453125, 0.045562744140625, 0.012786865234375, + -0.0233917236328125, -0.025146484375, -0.00409698486328125, + -0.019775390625, 0.00835418701171875, -0.0301971435546875, + 0.041046142578125, 0.012481689453125, -0.0015058517456054688, + 0.0059967041015625, -0.00667572021484375, 0.01248931884765625, 0.02105712890625, 0.04742431640625, 0.0295562744140625, - -0.04473876953125, 0.053680419921875, 0.0214385986328125, - -0.01190185546875, 0.003604888916015625, 0.004787445068359375, - 0.03082275390625, -0.00344085693359375, 0.0184173583984375, - -0.0284271240234375, 0.05914306640625, -0.0222625732421875, - -0.0287017822265625, 0.02880859375, -0.035003662109375, - -0.022186279296875, 0.0284881591796875, 0.0223388671875, - 0.0144500732421875, 0.043853759765625, 0.0281829833984375, - -0.047088623046875, -0.03582763671875, 0.04150390625, - 0.003925323486328125, 0.00934600830078125, -0.022003173828125, - -0.004741668701171875, -0.01403045654296875, - -0.04925537109375, -0.0005216598510742188, - -0.0065765380859375, 0.01715087890625, -0.0152130126953125, - 0.0200042724609375, -0.0130615234375, -0.0202484130859375, - -0.0025310516357421875, 0.0059661865234375, - 0.0185699462890625, -0.0101776123046875, -0.021728515625, - -0.01114654541015625, -0.0189361572265625, 0.0223541259765625, - 0.0265960693359375, -0.02960205078125, 0.019287109375, - -0.01416778564453125, 0.0121307373046875, 0.034515380859375, - -0.033782958984375, 0.000728607177734375, -0.03411865234375, - 0.00010061264038085938, 0.0010194778442382812, - 0.0228271484375, 0.009246826171875, -0.0114288330078125, - -0.00408935546875, 0.00986480712890625, -0.016845703125, - -0.039886474609375, -0.00739288330078125, - 0.007213592529296875, 0.0256805419921875, 0.01123809814453125, - 0.009185791015625, -0.03607177734375, 0.0251617431640625, - -0.017852783203125, -0.007503509521484375, -0.037750244140625, - -0.02001953125, -0.01474761962890625, -0.0113372802734375, - 0.0115814208984375, -0.00036406517028808594, - -0.014190673828125, -0.0032482147216796875, - 0.0017681121826171875, -0.0238494873046875, - -0.025299072265625, -0.0113525390625, 0.01398468017578125, - 0.01377105712890625, 0.01200103759765625, 0.00659942626953125, - -0.0007700920104980469, -0.05926513671875, 0.0075531005859375, - -0.033935546875, 0.00881195068359375, 0.002941131591796875, - -0.02313232421875, 0.02215576171875, 0.06939697265625, - -0.0294647216796875, -0.0032405853271484375, - -0.00983428955078125, -0.023223876953125, - -0.00310516357421875, -0.0014495849609375, - 0.01334381103515625, -0.0032253265380859375, 0.033447265625, - -0.0108642578125, 0.020355224609375, 0.0001189112663269043, - -0.0002608299255371094, -0.0404052734375, - 0.0038318634033203125, 0.0113983154296875, - -0.0255584716796875, -0.00572967529296875, 0.0184783935546875, - -0.0272064208984375, 0.01013946533203125, - -0.01021575927734375, -0.0146636962890625, - -0.0161285400390625, -0.022796630859375, 0.026214599609375, - -0.036590576171875, -0.0227813720703125, 0.0120086669921875, - 0.03369140625, -0.04632568359375, -0.012939453125, - -0.03302001953125, -0.035858154296875, 0.0019168853759765625, - -0.006450653076171875, 0.0162506103515625, 0.0193023681640625, - -0.00664520263671875, -0.01361083984375, 0.04168701171875, - -0.047821044921875, 0.011138916015625, -0.0188751220703125, - 0.00011795759201049805, -0.0157470703125, -0.01470947265625, - -0.01255035400390625, 0.0099334716796875, -0.0283355712890625, - 0.053131103515625, -0.02044677734375, -0.027008056640625, - -0.05059814453125, -0.01421356201171875, - 0.0009927749633789062, 0.0079803466796875, 0.0181884765625, - 0.0401611328125, 0.01983642578125, -0.028778076171875, - -0.004673004150390625, 0.0011386871337890625, - 0.0002682209014892578, -0.00746917724609375, - -0.038360595703125, -0.01195526123046875, -0.037689208984375, - 0.039764404296875, -0.0227813720703125, -0.00922393798828125, - -0.038116455078125, 0.0167388916015625, 0.01528167724609375, - -0.0020923614501953125, -0.0123291015625, - -0.007373809814453125, -0.04132080078125, 0.03277587890625, - -0.0265045166015625, 0.033172607421875, 0.0306549072265625, - -0.0022735595703125, 0.0080718994140625, -0.0269317626953125, - 0.0202178955078125, 0.0038299560546875, 0.022674560546875, - 0.00959014892578125, 0.039520263671875, 0.00336456298828125, - -0.0233917236328125, 0.0082855224609375, 0.01971435546875, - -0.035430908203125, 0.05047607421875, 0.006145477294921875, - -0.05767822265625, 0.00943756103515625, -0.005275726318359375, - 0.0142822265625, -0.012359619140625, -0.007305145263671875, - 0.006893157958984375, -0.023101806640625, 0.050323486328125, - -0.0291595458984375, 0.0227203369140625, -0.035614013671875, - 0.0396728515625, 0.00412750244140625, 0.034393310546875, - 0.00354766845703125, -0.032135009765625, -0.0251617431640625, - 0.003032684326171875, -0.0310211181640625, -0.033660888671875, - 0.0011720657348632812, -0.009918212890625, - -0.0187530517578125, -0.038116455078125, -0.0288543701171875, - 0.05987548828125, 0.005603790283203125, -0.032379150390625, - 0.0026187896728515625, 0.01201629638671875, - 0.001361846923828125, 0.0191497802734375, -0.00958251953125, - 0.01006317138671875, -0.01715087890625, -0.0546875, - -0.00728607177734375, 0.0243072509765625, 0.043853759765625, - -0.004398345947265625, -0.0305938720703125, - -0.0135650634765625, -0.0357666015625, -0.01088714599609375, - 0.01580810546875, 0.0311126708984375, 0.01107025146484375, - 0.0009198188781738281, 0.01346588134765625, - -0.00598907470703125, -0.031585693359375, 0.0192718505859375, - -0.0298919677734375, 0.024322509765625, -0.0038299560546875, - -0.03668212890625, -0.03363037109375, 0.01568603515625, - 0.027618408203125, -0.0020351409912109375, 0.029083251953125, - 0.0151824951171875, -0.025604248046875, -0.01169586181640625, - -0.003437042236328125, -0.030426025390625, -0.004547119140625, - -0.027496337890625, -0.017486572265625, 0.016510009765625, - 0.049224853515625, -0.044769287109375, 0.0099945068359375, - 0.03118896484375, -0.033172607421875, 0.018585205078125, - -0.032806396484375, 0.02734375, -0.03875732421875, - -0.05328369140625, 0.0248870849609375, 0.0003790855407714844, - -0.0165863037109375, -0.0008473396301269531, - 0.006267547607421875, -0.0216522216796875, - -0.004703521728515625, 0.0313720703125, -0.004276275634765625, - -0.0088653564453125, -0.01087188720703125, 0.032623291015625, - -0.046142578125, -0.0183563232421875, -0.00930023193359375, - 0.040618896484375, 0.006988525390625, 0.024169921875, - 0.007266998291015625, 0.013397216796875, 0.00238800048828125, - 0.00870513916015625, 0.02020263671875, -0.033050537109375, - 0.054840087890625, 0.023162841796875, -0.0002655982971191406, - -0.050750732421875, -0.023345947265625, -0.01496124267578125, - 0.02587890625, 0.00466156005859375, -0.01174163818359375, - 0.0265960693359375, -0.0006241798400878906, - -0.0022716522216796875, -0.0010480880737304688, - -0.01158905029296875, 0.0088043212890625, 0.01372528076171875, - -0.0122833251953125, 0.0022296905517578125, - -0.0183258056640625, 0.0221405029296875, 0.0283355712890625, - 0.042083740234375, -0.00858306884765625, -0.034881591796875, - 0.0240020751953125, 0.037689208984375, -0.01555633544921875, - -0.039154052734375, 0.05511474609375, 0.01129150390625, - 0.0042266845703125, 0.0019359588623046875, - -0.0081329345703125, 0.01029205322265625, - -0.002567291259765625, -0.0022258758544921875, - -0.0098876953125, -0.013153076171875, -0.035858154296875, - 0.01329803466796875, -0.01081085205078125, 0.0192718505859375, - 0.00901031494140625, 0.00010645389556884766, - 0.0134429931640625, -0.0038509368896484375, 0.01763916015625, - 0.0283355712890625, 0.01259613037109375, 0.00826263427734375, - 0.04632568359375, -0.054901123046875, -0.051239013671875, - 0.030975341796875, 0.0223846435546875, 0.0031890869140625, - -0.01381683349609375, -0.0230712890625, -0.00873565673828125, - 0.036590576171875, -0.010467529296875, -0.03173828125, - -0.004344940185546875, -0.03387451171875, - -0.00930023193359375, 0.0092620849609375, -0.0279541015625, - 0.0113372802734375, 0.0222930908203125, 0.0179901123046875, - -0.006481170654296875, -0.01105499267578125, - -0.005947113037109375, 0.005802154541015625, -0.0125732421875, - -0.00921630859375, -0.0013303756713867188, - -0.0197906494140625, -0.005924224853515625, - -0.0263214111328125, -0.01259613037109375, - -0.005962371826171875, 0.017303466796875, -0.0138397216796875, - -0.042022705078125, 0.0295562744140625, -0.015716552734375, - 0.0291748046875, 0.035552978515625, 0.00788116455078125, - -0.01380157470703125, -0.0011653900146484375, - -0.0090179443359375, -0.001216888427734375, - 0.0025177001953125, 0.0003676414489746094, 0.0197906494140625, - -0.02069091796875, 0.01342010498046875, 0.0030536651611328125, - 0.03173828125, -0.001850128173828125, -0.052215576171875, - -0.023193359375, -0.012176513671875, -0.0166778564453125, - -0.03997802734375, -0.041229248046875, -0.017974853515625, - -0.00385284423828125, 0.005035400390625, - -0.002956390380859375, 0.033843994140625, -0.01116943359375, - 0.0279083251953125, -0.0178070068359375, 0.024627685546875, - 0.032867431640625, 0.003444671630859375, 0.0240478515625, - -0.01445770263671875, 0.040771484375, -0.01230621337890625, - 0.00666046142578125, -0.008087158203125, 0.006805419921875, - -0.004276275634765625, 0.0221405029296875, -0.024749755859375, - 0.0400390625, 0.0138397216796875, 0.0361328125, - -0.0028705596923828125, -0.003787994384765625, - 0.06475830078125, 0.0302886962890625, -0.043304443359375, - -0.04559326171875, 0.020751953125, -0.0275115966796875, - -0.02569580078125, -0.01329803466796875, 0.003986358642578125, - 0.04998779296875, -0.0101776123046875, -0.0172119140625, - 0.048004150390625, 0.014007568359375, -0.0207061767578125, - -0.0168914794921875, 0.0146026611328125, 0.0286712646484375, - 0.03729248046875, -0.0244598388671875, 0.00667572021484375, - -0.0207366943359375, 0.03302001953125, 0.00540924072265625, - -0.0284576416015625, 0.01849365234375, -0.011444091796875, - 0.001476287841796875, -0.005615234375, 0.03778076171875, - 0.02642822265625, 0.01239776611328125, -0.01181793212890625, - 0.03521728515625, 0.013031005859375, 0.01139068603515625, - 0.00672149658203125, 0.024444580078125, 0.0197601318359375, - -0.0177001953125, -0.024200439453125, 0.0216827392578125, - -0.0004317760467529297, 0.00811767578125, -0.0131683349609375, - 0.0012493133544921875, -0.01227569580078125, - 0.0030918121337890625, 0.041107177734375, 0.012481689453125, - -0.029998779296875, -0.0092620849609375, -0.01904296875, - 0.0090484619140625, -0.0301055908203125, -0.00742340087890625, - -0.01364898681640625, 0.0297088623046875, 0.0328369140625, - -0.0116729736328125, 0.00152587890625, 0.045867919921875, - 0.0035400390625, -0.03497314453125, 0.023406982421875, - 0.03289794921875, -0.028076171875, 0.01800537109375, - 0.01024627685546875, -0.004169464111328125, -0.00396728515625, - -0.0245513916015625, -0.01021575927734375, -0.031585693359375, - -0.05474853515625, 0.0168304443359375, 0.026519775390625, - 0.0267333984375, -0.01230621337890625, -0.0149383544921875, - 0.0204620361328125, 0.015380859375, -0.0626220703125, - -0.0162353515625, 0.0220794677734375, -0.0171356201171875, - -0.01561737060546875, -0.035980224609375, 0.0192718505859375, + -0.04473876953125, 0.05364990234375, 0.021392822265625, + -0.01189422607421875, 0.003597259521484375, + 0.004817962646484375, 0.03076171875, -0.0034389495849609375, + 0.0184326171875, -0.0283966064453125, 0.05914306640625, + -0.0222625732421875, -0.0287017822265625, 0.028778076171875, + -0.0350341796875, -0.0222015380859375, 0.02850341796875, + 0.022308349609375, 0.01444244384765625, 0.04388427734375, + 0.028167724609375, -0.04718017578125, -0.03582763671875, + 0.04150390625, 0.003902435302734375, 0.0093536376953125, + -0.02203369140625, -0.004726409912109375, -0.0140228271484375, + -0.049285888671875, -0.0005688667297363281, -0.006591796875, + 0.0171356201171875, -0.01517486572265625, 0.0199432373046875, + -0.013031005859375, -0.0202789306640625, + -0.002544403076171875, 0.005962371826171875, 0.0185546875, + -0.01016998291015625, -0.021697998046875, + -0.01117706298828125, -0.0189361572265625, 0.0223541259765625, + 0.0265960693359375, -0.0296173095703125, 0.019287109375, + -0.0141754150390625, 0.01216888427734375, 0.034698486328125, + -0.033782958984375, 0.0007343292236328125, -0.034149169921875, + 0.0001068115234375, 0.0009908676147460938, 0.0228271484375, + 0.00920867919921875, -0.01145172119140625, + -0.004093170166015625, 0.00984954833984375, + -0.0168609619140625, -0.039886474609375, + -0.007373809814453125, 0.007198333740234375, + 0.025665283203125, 0.01123046875, 0.0092315673828125, + -0.03607177734375, 0.0251922607421875, -0.01788330078125, + -0.007472991943359375, -0.03778076171875, -0.0200042724609375, + -0.01473236083984375, -0.01132965087890625, 0.0115966796875, + -0.0003600120544433594, -0.014190673828125, + -0.0032367706298828125, 0.0017652511596679688, + -0.023895263671875, -0.02532958984375, -0.0113525390625, + 0.0139923095703125, 0.01377105712890625, 0.01204681396484375, + 0.00659942626953125, -0.0007576942443847656, + -0.05926513671875, 0.007549285888671875, -0.033905029296875, + 0.00885009765625, 0.00292205810546875, -0.023193359375, + 0.022186279296875, 0.06939697265625, -0.02947998046875, + -0.0032253265380859375, -0.009796142578125, + -0.0232391357421875, -0.0031147003173828125, + -0.0014448165893554688, 0.0133056640625, + -0.003246307373046875, 0.033447265625, -0.0108184814453125, + 0.02032470703125, 0.00016582012176513672, + -0.0002765655517578125, -0.0404052734375, 0.00383758544921875, + 0.01140594482421875, -0.025543212890625, + -0.005756378173828125, 0.0184478759765625, -0.027191162109375, + 0.010162353515625, -0.01020050048828125, -0.0146636962890625, + -0.01611328125, -0.0227813720703125, 0.0262298583984375, + -0.036590576171875, -0.0228118896484375, 0.01200103759765625, + 0.03369140625, -0.04632568359375, -0.012908935546875, + -0.033050537109375, -0.035858154296875, 0.001895904541015625, + -0.00644683837890625, 0.0162811279296875, 0.0193023681640625, + -0.00662994384765625, -0.01361846923828125, 0.04168701171875, + -0.0478515625, 0.01113128662109375, -0.018890380859375, + 0.00014603137969970703, -0.015716552734375, + -0.01471710205078125, -0.01258087158203125, 0.00994873046875, + -0.028350830078125, 0.05316162109375, -0.02044677734375, + -0.0269927978515625, -0.0506591796875, -0.0142364501953125, + 0.0009512901306152344, 0.00801849365234375, + 0.0181732177734375, 0.04010009765625, 0.01983642578125, + -0.028778076171875, -0.004695892333984375, + 0.0011377334594726562, 0.0002841949462890625, + -0.007450103759765625, -0.038360595703125, + -0.01195526123046875, -0.037689208984375, 0.03973388671875, + -0.0227813720703125, -0.0092620849609375, -0.038116455078125, + 0.0167388916015625, 0.01529693603515625, + -0.002117156982421875, -0.01233673095703125, + -0.00740814208984375, -0.04132080078125, 0.03277587890625, + -0.026519775390625, 0.033172607421875, 0.0306549072265625, + -0.0022640228271484375, 0.0081024169921875, + -0.0269317626953125, 0.020233154296875, 0.0038509368896484375, + 0.0226898193359375, 0.00959014892578125, 0.039459228515625, + 0.003368377685546875, -0.0233306884765625, 0.0083160400390625, + 0.01971435546875, -0.035430908203125, 0.0504150390625, + 0.006153106689453125, -0.05767822265625, 0.00940704345703125, + -0.005260467529296875, 0.01422882080078125, + -0.0123748779296875, -0.00731658935546875, + 0.006916046142578125, -0.0230865478515625, 0.05035400390625, + -0.0291595458984375, 0.022705078125, -0.03564453125, + 0.039642333984375, 0.004154205322265625, 0.03436279296875, + 0.0035457611083984375, -0.0321044921875, -0.025146484375, + 0.003063201904296875, -0.030975341796875, -0.033599853515625, + 0.0011835098266601562, -0.00994873046875, -0.018798828125, + -0.03802490234375, -0.0288848876953125, 0.05987548828125, + 0.005603790283203125, -0.032440185546875, + 0.0026149749755859375, 0.011993408203125, + 0.001346588134765625, 0.0190887451171875, -0.009613037109375, + 0.01007080078125, -0.0171356201171875, -0.0546875, + -0.007312774658203125, 0.0242767333984375, 0.04388427734375, + -0.00441741943359375, -0.030609130859375, -0.0135498046875, + -0.0357666015625, -0.0108795166015625, 0.0157928466796875, + 0.0311126708984375, 0.01108551025390625, + 0.0009121894836425781, 0.01349639892578125, + -0.005985260009765625, -0.031585693359375, 0.019256591796875, + -0.029876708984375, 0.02435302734375, -0.0038127899169921875, + -0.036712646484375, -0.033660888671875, 0.0157012939453125, + 0.027618408203125, -0.0020122528076171875, 0.0290679931640625, + 0.01519775390625, -0.025604248046875, -0.01172637939453125, + -0.003452301025390625, -0.0303955078125, -0.00453948974609375, + -0.027496337890625, -0.01751708984375, 0.0164794921875, + 0.049224853515625, -0.044769287109375, 0.0099639892578125, + 0.03118896484375, -0.033203125, 0.0185546875, + -0.03277587890625, 0.02734375, -0.038726806640625, + -0.053253173828125, 0.024932861328125, 0.00039267539978027344, + -0.016571044921875, -0.0008702278137207031, + 0.006275177001953125, -0.021728515625, -0.004669189453125, + 0.031402587890625, -0.0042877197265625, -0.0088653564453125, + -0.010894775390625, 0.032623291015625, -0.046173095703125, + -0.0183563232421875, -0.0092620849609375, 0.040618896484375, + 0.006977081298828125, 0.0242156982421875, + 0.007274627685546875, 0.0134124755859375, 0.00235748291015625, + 0.00870513916015625, 0.0201873779296875, -0.033050537109375, + 0.05487060546875, 0.0231170654296875, -0.0002980232238769531, + -0.050750732421875, -0.0233917236328125, -0.01499176025390625, + 0.02587890625, 0.00461578369140625, -0.01174163818359375, + 0.026611328125, -0.0006389617919921875, -0.002239227294921875, + -0.0010595321655273438, -0.01160430908203125, + 0.008819580078125, 0.01372528076171875, -0.01226806640625, + 0.002239227294921875, -0.0183258056640625, 0.0221405029296875, + 0.0283203125, 0.0421142578125, -0.00855255126953125, + -0.034881591796875, 0.0240020751953125, 0.0377197265625, + -0.0155029296875, -0.03912353515625, 0.055145263671875, + 0.011260986328125, 0.004222869873046875, + 0.0019702911376953125, -0.0081024169921875, + 0.0102691650390625, -0.0025806427001953125, + -0.0022258758544921875, -0.00988006591796875, + -0.013153076171875, -0.035858154296875, 0.0132598876953125, + -0.0107879638671875, 0.019256591796875, 0.009033203125, + 0.00010395050048828125, 0.01342010498046875, -0.00390625, + 0.0177001953125, 0.028350830078125, 0.0125732421875, + 0.00830841064453125, 0.04632568359375, -0.054901123046875, + -0.051239013671875, 0.031005859375, 0.02239990234375, + 0.0031948089599609375, -0.01383209228515625, -0.0230712890625, + -0.00870513916015625, 0.03656005859375, -0.01050567626953125, + -0.031768798828125, -0.00438690185546875, -0.033843994140625, + -0.00930023193359375, 0.00926971435546875, -0.0279541015625, + 0.01134490966796875, 0.02227783203125, 0.017974853515625, + -0.006488800048828125, -0.0110931396484375, + -0.005939483642578125, 0.005809783935546875, -0.0125732421875, + -0.0092620849609375, -0.0013303756713867188, + -0.0197906494140625, -0.005931854248046875, -0.02630615234375, + -0.01263427734375, -0.006008148193359375, 0.017303466796875, + -0.0138702392578125, -0.0419921875, 0.0295562744140625, + -0.015716552734375, 0.0291748046875, 0.0355224609375, + 0.00782012939453125, -0.013824462890625, + -0.0011453628540039062, -0.009033203125, + -0.0012044906616210938, 0.002529144287109375, + 0.0003771781921386719, 0.01983642578125, -0.0207061767578125, + 0.01340484619140625, 0.003055572509765625, 0.031707763671875, + -0.0019130706787109375, -0.05224609375, -0.023223876953125, + -0.012176513671875, -0.01666259765625, -0.040008544921875, + -0.041229248046875, -0.01800537109375, -0.0038776397705078125, + 0.005023956298828125, -0.0029544830322265625, + 0.03387451171875, -0.011199951171875, 0.0279388427734375, + -0.01776123046875, 0.0246124267578125, 0.032867431640625, + 0.0034275054931640625, 0.0240325927734375, -0.014434814453125, + 0.040771484375, -0.0123138427734375, 0.006633758544921875, + -0.00806427001953125, 0.006805419921875, -0.00426483154296875, + 0.0221405029296875, -0.024749755859375, 0.040069580078125, + 0.013824462890625, 0.0361328125, -0.0028667449951171875, + -0.0037746429443359375, 0.06475830078125, 0.030303955078125, + -0.043304443359375, -0.04559326171875, 0.0207366943359375, + -0.0275115966796875, -0.0256805419921875, -0.01324462890625, + 0.004016876220703125, 0.050048828125, -0.01021575927734375, + -0.0172119140625, 0.048004150390625, 0.0139617919921875, + -0.0207061767578125, -0.0168914794921875, 0.0146026611328125, + 0.0286407470703125, 0.037322998046875, -0.0244598388671875, + 0.006633758544921875, -0.0207366943359375, 0.03302001953125, + 0.005397796630859375, -0.0284423828125, 0.0184783935546875, + -0.01146697998046875, 0.0014848709106445312, + -0.005641937255859375, 0.03778076171875, 0.0264434814453125, + 0.0123748779296875, -0.01178741455078125, 0.03521728515625, + 0.013092041015625, 0.01140594482421875, 0.00672149658203125, + 0.024505615234375, 0.019775390625, -0.0177154541015625, + -0.0242156982421875, 0.02166748046875, -0.0004470348358154297, + 0.00811004638671875, -0.013153076171875, + 0.0012683868408203125, -0.01226806640625, + 0.003078460693359375, 0.041107177734375, 0.0124969482421875, + -0.030029296875, -0.0092620849609375, -0.01904296875, + 0.00907135009765625, -0.0301055908203125, + -0.007450103759765625, -0.01367950439453125, + 0.0297698974609375, 0.032867431640625, -0.0117034912109375, + 0.0015239715576171875, 0.045867919921875, 0.00350189208984375, + -0.034942626953125, 0.023406982421875, 0.03289794921875, + -0.0280914306640625, 0.018035888671875, 0.010223388671875, + -0.00412750244140625, -0.003978729248046875, + -0.024566650390625, -0.01024627685546875, -0.031585693359375, + -0.05474853515625, 0.016815185546875, 0.0265045166015625, + 0.02679443359375, -0.01230621337890625, -0.014892578125, + 0.02044677734375, 0.0153656005859375, -0.06256103515625, + -0.0162506103515625, 0.0221099853515625, -0.017120361328125, + -0.015594482421875, -0.035980224609375, 0.0192718505859375, 0.043701171875, 0.0413818359375, 0.03314208984375, - 0.00016868114471435547, -0.000579833984375, - -0.0177154541015625, -0.037384033203125, 0.0377197265625, - 0.0264739990234375, 0.01340484619140625, 0.02313232421875, - 0.04119873046875, -0.0037288665771484375, -0.0226287841796875, - -0.0005016326904296875, 0.00885772705078125, - -0.0029125213623046875, 0.007633209228515625, 0.00732421875, - -0.0113983154296875, 0.00392913818359375, - -0.00774383544921875, -0.040008544921875, -0.0231781005859375, - 0.007091522216796875, -0.0286865234375, 0.032440185546875, - 0.0177001953125, -0.015167236328125, 0.0082550048828125, - -0.01465606689453125, 0.01073455810546875, - 0.007236480712890625, 0.00146484375, 0.03265380859375, - -0.0084686279296875, 0.0301361083984375, 0.0013427734375, - -0.01102447509765625, -0.0197906494140625, - 0.01364898681640625, -0.031219482421875, -0.0229034423828125, - 0.0275115966796875, 0.0003364086151123047, 0.050811767578125, - 0.0234222412109375, 0.01097869873046875, -0.017974853515625, - -0.0154876708984375, 0.016876220703125, 0.0070037841796875, - 0.00920867919921875, -0.0135040283203125, -0.019073486328125, - 0.028564453125, 0.00928497314453125, 0.030181884765625, - -0.01273345947265625, 0.00492095947265625, - 0.0024242401123046875, -0.06781005859375, -0.00714111328125, - 0.0021343231201171875, 0.003330230712890625, - -0.0007662773132324219, 0.019500732421875, 0.0095062255859375, - -0.0262298583984375, -0.00960540771484375, - 0.006473541259765625, 0.05169677734375, -0.017730712890625, - 0.0238037109375, -0.0214691162109375, 0.0304412841796875, - -0.0126190185546875, -0.03070068359375, -0.037078857421875, - 0.022369384765625, 0.034515380859375, -0.0205841064453125, - 0.004116058349609375, -0.038787841796875, -0.0221710205078125, - -0.024993896484375, -0.021453857421875, - -0.0003190040588378906, 0.0036602020263671875, - 0.0107269287109375, 0.0152435302734375, -0.016265869140625, - -0.0222930908203125, 0.0105133056640625, -0.0185546875, - -0.015045166015625, 0.0134735107421875, 0.002307891845703125, - 0.007579803466796875, -0.0022735595703125, 0.0276336669921875, - 0.0083160400390625, 0.01552581787109375, - -0.007038116455078125, 0.008331298828125, -0.0135955810546875, - 0.020721435546875, 0.050933837890625, 0.0147552490234375, - 0.0205841064453125, -0.00490570068359375, 0.038299560546875, - 0.0301055908203125, 0.05523681640625, 0.00766754150390625, - 0.00782012939453125, -0.00164794921875, 0.0191497802734375, - 0.002674102783203125, -0.017242431640625, 0.02203369140625, - -0.0187835693359375, 0.0350341796875, -0.01007080078125, - -0.04803466796875, 0.01506805419921875, -0.031646728515625, - 0.0182342529296875, 0.0239105224609375, 0.0435791015625, - 0.0156097412109375, -0.0275115966796875, 0.0169525146484375, - -0.0036716461181640625, 0.0154571533203125, - -0.00377655029296875, -0.003070831298828125, - 0.034332275390625, -0.0034275054931640625, - -0.0285797119140625, -0.03607177734375, - -0.0008091926574707031, -0.0127410888671875, -0.036376953125, - -0.007793426513671875, -0.043792724609375, 0.0270538330078125, - -0.058013916015625, -0.01438140869140625, - 0.007251739501953125, 0.045013427734375, 0.00720977783203125, - 0.0254058837890625, 0.054718017578125, 0.0061798095703125, - -0.0198822021484375, 0.010162353515625, -0.035919189453125, - 0.0014743804931640625, -0.0060577392578125, -0.02752685546875, - -0.01453399658203125, -0.0137481689453125, - -0.00218963623046875, -0.0038661956787109375, - 0.006755828857421875, 0.00434112548828125, -0.020294189453125, - 0.0181884765625, -0.004611968994140625, -0.0200042724609375, - 0.034454345703125, -0.0709228515625, -0.028533935546875, - 0.0260772705078125, -0.044677734375, 0.021240234375, - 0.032470703125, 0.0162811279296875, -0.01093292236328125, + 0.0001302957534790039, -0.00060272216796875, + -0.0177154541015625, -0.037384033203125, 0.037750244140625, + 0.0265045166015625, 0.01338958740234375, 0.0231170654296875, + 0.04119873046875, -0.0037441253662109375, -0.0226898193359375, + -0.000499725341796875, 0.00890350341796875, + -0.002899169921875, 0.007648468017578125, + 0.007366180419921875, -0.0113983154296875, + 0.003940582275390625, -0.007781982421875, -0.0400390625, + -0.0231781005859375, 0.007110595703125, -0.0287017822265625, + 0.032440185546875, 0.0176849365234375, -0.01515960693359375, + 0.00826263427734375, -0.0146636962890625, 0.01073455810546875, + 0.007213592529296875, 0.0014743804931640625, 0.03265380859375, + -0.00846099853515625, 0.03009033203125, 0.0013189315795898438, + -0.01103973388671875, -0.01983642578125, 0.0136566162109375, + -0.0312347412109375, -0.0228729248046875, 0.0274658203125, + 0.00033664703369140625, 0.050811767578125, 0.0234222412109375, + 0.011016845703125, -0.01800537109375, -0.0154876708984375, + 0.016876220703125, 0.006988525390625, 0.00920867919921875, + -0.01349639892578125, -0.0190887451171875, 0.028564453125, + 0.0092926025390625, 0.03021240234375, -0.01274871826171875, + 0.0049285888671875, 0.0024471282958984375, -0.06781005859375, + -0.007171630859375, 0.002124786376953125, 0.00334930419921875, + -0.0007500648498535156, 0.01947021484375, 0.0095367431640625, + -0.0262298583984375, -0.00958251953125, 0.00647735595703125, + 0.051727294921875, -0.0177154541015625, 0.0238189697265625, + -0.021453857421875, 0.030426025390625, -0.01262664794921875, + -0.0307159423828125, -0.037078857421875, 0.0223541259765625, + 0.03448486328125, -0.020599365234375, 0.0041046142578125, + -0.038787841796875, -0.0221710205078125, -0.0249786376953125, + -0.0214385986328125, -0.0003192424774169922, + 0.0036640167236328125, 0.0107574462890625, 0.0152435302734375, + -0.0162811279296875, -0.0222930908203125, 0.01052093505859375, + -0.0185699462890625, -0.01507568359375, 0.013458251953125, + 0.0023326873779296875, 0.00759124755859375, + -0.0022640228271484375, 0.0276336669921875, + 0.0083465576171875, 0.0155181884765625, -0.00701904296875, + 0.00833892822265625, -0.01355743408203125, 0.02069091796875, + 0.050933837890625, 0.01476287841796875, 0.0205841064453125, + -0.004924774169921875, 0.038299560546875, 0.0301361083984375, + 0.055267333984375, 0.00765228271484375, 0.00782012939453125, + -0.0016546249389648438, 0.0191497802734375, + 0.0026912689208984375, -0.0172119140625, 0.0219879150390625, + -0.0187835693359375, 0.035064697265625, -0.010101318359375, + -0.048065185546875, 0.0150604248046875, -0.031707763671875, + 0.0182647705078125, 0.02392578125, 0.043548583984375, + 0.01558685302734375, -0.0275115966796875, 0.01690673828125, + -0.003688812255859375, 0.01544952392578125, + -0.0037899017333984375, -0.0030841827392578125, + 0.0343017578125, -0.003437042236328125, -0.0285797119140625, + -0.03607177734375, -0.0008149147033691406, -0.01275634765625, + -0.036376953125, -0.007778167724609375, -0.04376220703125, + 0.027099609375, -0.05804443359375, -0.0144195556640625, + 0.0072479248046875, 0.045013427734375, 0.007221221923828125, + 0.025390625, 0.054718017578125, 0.00618743896484375, + -0.0198974609375, 0.0101470947265625, -0.035919189453125, + 0.00145721435546875, -0.00609588623046875, -0.027557373046875, + -0.01454925537109375, -0.0137176513671875, -0.002197265625, + -0.0038280487060546875, 0.006763458251953125, + 0.00432586669921875, -0.0203094482421875, 0.0181732177734375, + -0.00463104248046875, -0.0200042724609375, 0.03448486328125, + -0.0709228515625, -0.028533935546875, 0.0260162353515625, + -0.044677734375, 0.0212249755859375, 0.032501220703125, + 0.0163116455078125, -0.01091766357421875, -0.01284027099609375, -0.028900146484375, - 0.0009975433349609375, -0.0187835693359375, - 0.0098419189453125, 0.0171966552734375, 0.030670166015625, - 0.00463104248046875, -0.022796630859375, 0.0117950439453125, - -0.028472900390625, -0.021148681640625, -0.039337158203125, - 0.0176544189453125, -0.038238525390625, -0.039581298828125, - 0.002685546875, -0.01483154296875, 0.00533294677734375, - 0.0000928044319152832, 0.01045989990234375, -0.01025390625, - -0.0207061767578125, -0.025665283203125, 0.0222015380859375, - -0.0111083984375, 0.023406982421875, -0.0209197998046875, - -0.0036716461181640625, -0.037689208984375, - 0.0189056396484375, 0.0112457275390625, -0.016204833984375, - 0.0017604827880859375, 0.01241302490234375, - -0.0038776397705078125, -0.00168609619140625, 0.043701171875, - -0.0297698974609375, 0.005054473876953125, - -0.0214385986328125, -0.026611328125, -0.01385498046875, - 0.00682830810546875, -0.0007939338684082031, - 0.030181884765625, 0.051513671875, -0.04296875, - -0.037811279296875, -0.0005049705505371094, - 0.01035308837890625, 0.0136260986328125, 0.025604248046875, - 0.0211334228515625, -0.01605224609375, -0.0228118896484375, - -0.004764556884765625, 0.01377105712890625, - -0.01389312744140625, 0.0256805419921875, + 0.0009822845458984375, -0.0187835693359375, + 0.0098114013671875, 0.0172119140625, 0.0307159423828125, + 0.004657745361328125, -0.0228271484375, 0.01180267333984375, + -0.028472900390625, -0.0211639404296875, -0.03936767578125, + 0.0176239013671875, -0.038238525390625, -0.03961181640625, + 0.0026645660400390625, -0.01483154296875, + 0.005336761474609375, 0.00011247396469116211, + 0.01042938232421875, -0.01023101806640625, -0.020721435546875, + -0.0256500244140625, 0.022186279296875, -0.01110076904296875, + 0.0234222412109375, -0.0209197998046875, + -0.0036983489990234375, -0.037628173828125, + 0.0189056396484375, 0.0112762451171875, -0.0162200927734375, + 0.00176239013671875, 0.01241302490234375, + -0.0038700103759765625, -0.0017251968383789062, + 0.043731689453125, -0.0298004150390625, 0.0050811767578125, + -0.0214385986328125, -0.0266265869140625, + -0.01386260986328125, 0.006877899169921875, + -0.0007877349853515625, 0.03021240234375, 0.051544189453125, + -0.04302978515625, -0.037811279296875, -0.0004911422729492188, + 0.0103607177734375, 0.0136260986328125, 0.025604248046875, + 0.0211639404296875, -0.0160675048828125, -0.022796630859375, + -0.004734039306640625, 0.013763427734375, + -0.01389312744140625, 0.0256195068359375, 0.0035343170166015625, -0.026397705078125, - -0.0199737548828125, -0.018646240234375, -0.025054931640625, - 0.02752685546875, 0.01459503173828125, -0.002986907958984375, - 0.038421630859375, -0.0166168212890625, 0.00832366943359375, - 0.06048583984375, 0.0234527587890625, 0.01180267333984375, - 0.00638580322265625, -0.0140838623046875, -0.0074615478515625, - -0.00616455078125, -0.02105712890625, 0.000720977783203125, - -0.01064300537109375, 0.00774383544921875, 0.016632080078125, - 0.0108795166015625, -0.0135498046875, 0.00914764404296875, - 0.034332275390625, -0.01390838623046875, -0.037353515625, - -0.0296173095703125, 0.01849365234375, -0.0230255126953125, - 0.01108551025390625, -0.0214691162109375, 0.0247955322265625, - -0.00872802734375, 0.0184478759765625, 0.00928497314453125, - 0.0202484130859375, 0.0223541259765625, 0.006092071533203125, - 0.035430908203125, -0.035614013671875, -0.0004284381866455078, - -0.0036907196044921875, 0.0053863525390625, - 0.002460479736328125, 0.00722503662109375, - -0.0257110595703125, 0.00926971435546875, 0.00696563720703125, - -0.007740020751953125, 0.01490020751953125, -0.03143310546875, - 0.0160980224609375, -0.0022335052490234375, 0.0146484375, - 0.01129913330078125, 0.019683837890625, -0.0193634033203125, - 0.0092620849609375, 0.01381683349609375, 0.001338958740234375, - 0.01309967041015625, -0.0032100677490234375, - -0.00939178466796875, 0.02569580078125, 0.030731201171875, - 0.023895263671875, 0.0019083023071289062, -0.0124969482421875, - -0.01023101806640625, -0.03558349609375, 0.022857666015625, - -0.000028312206268310547, 0.0123291015625, 0.005157470703125, - -0.01056671142578125, 0.005367279052734375, 0.009490966796875, - 0.005687713623046875, -0.00585174560546875, - 0.01279449462890625, -0.01137542724609375, - 0.005809783935546875, 0.0124359130859375, - -0.006900787353515625, 0.04034423828125, -0.0111083984375, - -0.0012426376342773438, 0.0159454345703125, -0.0450439453125, - -0.01468658447265625, 0.00832366943359375, - -0.01213836669921875, -0.0028018951416015625, - 0.0014476776123046875, -0.00453948974609375, - -0.003498077392578125, -0.0117645263671875, - 0.00293731689453125, -0.0258941650390625, 0.008209228515625, - -0.00203704833984375, 0.0144195556640625, 0.036346435546875, - -0.0119476318359375, -0.03778076171875, 0.0245819091796875, - -0.035858154296875, 0.0084991455078125, 0.01824951171875, - 0.0088653564453125, -0.0467529296875, 0.01412200927734375, - 0.01203155517578125, -0.0232391357421875, - -0.00788116455078125, 0.0123443603515625, -0.031036376953125, - 0.003704071044921875, 0.0136260986328125, - 0.0031147003173828125, 0.0007843971252441406, - 0.00490570068359375 + -0.0199432373046875, -0.018646240234375, -0.0250701904296875, + 0.0275421142578125, 0.0146026611328125, -0.002964019775390625, + 0.03839111328125, -0.0166168212890625, 0.00832366943359375, + 0.060455322265625, 0.0234527587890625, 0.0117950439453125, + 0.006397247314453125, -0.014068603515625, + -0.007488250732421875, -0.00616455078125, -0.0210723876953125, + 0.0007615089416503906, -0.01064300537109375, 0.00775146484375, + 0.016632080078125, 0.01084136962890625, -0.01355743408203125, + 0.0091552734375, 0.03436279296875, -0.013916015625, + -0.037353515625, -0.029632568359375, 0.018524169921875, + -0.0230712890625, 0.01107025146484375, -0.021484375, + 0.0247955322265625, -0.00873565673828125, 0.0184783935546875, + 0.00930023193359375, 0.0202484130859375, 0.0223541259765625, + 0.00608062744140625, 0.035430908203125, -0.035614013671875, + -0.0004277229309082031, -0.003662109375, 0.00539398193359375, + 0.0024127960205078125, 0.007221221923828125, + -0.0256805419921875, 0.00926971435546875, 0.00698089599609375, + -0.0077667236328125, 0.014862060546875, -0.031402587890625, + 0.0160675048828125, -0.0022525787353515625, + 0.01468658447265625, 0.01129913330078125, 0.019622802734375, + -0.0193939208984375, 0.00928497314453125, 0.013824462890625, + 0.0013227462768554688, 0.0131072998046875, + -0.0032062530517578125, -0.0093841552734375, + 0.0257415771484375, 0.0307464599609375, 0.02386474609375, + 0.0018949508666992188, -0.01251220703125, -0.010223388671875, + -0.035614013671875, 0.0228118896484375, + -0.000028192996978759766, 0.0123138427734375, 0.005126953125, + -0.010589599609375, 0.00533294677734375, 0.009490966796875, + 0.00566864013671875, -0.005893707275390625, + 0.01282501220703125, -0.0113372802734375, 0.00579071044921875, + 0.01239776611328125, -0.0068359375, 0.04034423828125, + -0.011077880859375, -0.0012264251708984375, 0.015899658203125, + -0.0450439453125, -0.01468658447265625, 0.0083160400390625, + -0.0121612548828125, -0.0027618408203125, + 0.0014629364013671875, -0.004535675048828125, + -0.0034885406494140625, -0.01180267333984375, 0.0029296875, + -0.025909423828125, 0.00821685791015625, + -0.0020427703857421875, 0.014404296875, 0.036346435546875, + -0.01197052001953125, -0.03778076171875, 0.0245819091796875, + -0.035858154296875, 0.00846099853515625, 0.0182342529296875, + 0.0088653564453125, -0.0467529296875, 0.01409912109375, + 0.01203155517578125, -0.0232696533203125, -0.0079498291015625, + 0.0123291015625, -0.0310516357421875, 0.00372314453125, + 0.0136260986328125, 0.00312042236328125, + 0.0007691383361816406, 0.004894256591796875 ], "index": 0, "object": "embedding" @@ -859,13 +862,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7dabef7c2d4-VIE", + "cf-ray": "9fdb11ff9b3c34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:36:08 GMT", + "date": "Mon, 18 May 2026 13:04:53 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "68", + "openai-processing-ms": "50", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -880,7 +883,7 @@ "x-ratelimit-remaining-tokens": "9999993", "x-ratelimit-reset-requests": "6ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_0242fde0bef542c7b90930a90ed938c2" + "x-request-id": "req_d522ff6336064007bede1003233679fb" }, "status": 200, "statusText": "OK" @@ -890,7 +893,7 @@ "callIndex": 3, "id": "9681fdcd86301e38", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:09.094Z", + "recordedAt": "2026-05-18T13:04:54.420Z", "request": { "body": { "kind": "json", @@ -956,19 +959,19 @@ "arguments": "{\"location\":\"Paris, France\"}", "name": "get_weather" }, - "id": "call_P6lWgTJAEidjiuaR8dr3WPTI", + "id": "call_7Vc2ttu5StELTNtddXg2SAb1", "type": "function" } ] } } ], - "created": 1778844968, - "id": "chatcmpl-Dfl7oVxrA28zifY2NwYZRehdPCEEt", + "created": 1779109493, + "id": "chatcmpl-DgrwLwxk3TOL6iMVJF4UzBIgVQthg", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_af421e7311", + "system_fingerprint": "fp_9ee1f7f450", "usage": { "completion_tokens": 16, "completion_tokens_details": { @@ -990,13 +993,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7dbb8c0c2d4-VIE", + "cf-ray": "9fdb12003d5e34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:36:09 GMT", + "date": "Mon, 18 May 2026 13:04:54 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "651", + "openai-processing-ms": "495", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1008,10 +1011,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999952", + "x-ratelimit-remaining-tokens": "149999950", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_318e22e9e9144a41b2dd316fbebce0e6" + "x-request-id": "req_bb907c4758c74910a5bf229645b91084" }, "status": 200, "statusText": "OK" @@ -1021,7 +1024,7 @@ "callIndex": 4, "id": "d2e0c4c7403336d9", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:09.765Z", + "recordedAt": "2026-05-18T13:04:54.962Z", "request": { "body": { "kind": "json", @@ -1087,19 +1090,19 @@ "arguments": "{\"city\":\"Paris\"}", "name": "json" }, - "id": "call_D5jLT9gE0FHJyIBOGB2py5on", + "id": "call_aaFUHPmh95Iz9JzWLZr16gT5", "type": "function" } ] } } ], - "created": 1778844969, - "id": "chatcmpl-Dfl7pRB5STpBSHPufHYKXLb8DJxM7", + "created": 1779109494, + "id": "chatcmpl-DgrwMFRNE9vWSPeEYAjdsjhTP6mwb", "model": "gpt-4o-mini-2024-07-18", "object": "chat.completion", "service_tier": "default", - "system_fingerprint": "fp_5b8b54db47", + "system_fingerprint": "fp_54f26dc974", "usage": { "completion_tokens": 5, "completion_tokens_details": { @@ -1121,13 +1124,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7e17f86c2d4-VIE", + "cf-ray": "9fdb12049b8b34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:36:09 GMT", + "date": "Mon, 18 May 2026 13:04:55 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "451", + "openai-processing-ms": "334", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1139,10 +1142,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-remaining-tokens": "149999987", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_838a6f7e3f5b4bd9aac505e1c2e69396" + "x-request-id": "req_c78358bb3df549f0b888c68ed4597042" }, "status": 200, "statusText": "OK" @@ -1152,7 +1155,7 @@ "callIndex": 5, "id": "c505f9dbd2a1ee9b", "matchKey": "POST api.openai.com/v1/chat/completions", - "recordedAt": "2026-05-15T11:36:10.696Z", + "recordedAt": "2026-05-18T13:04:55.721Z", "request": { "body": { "kind": "json", @@ -1202,13 +1205,13 @@ "response": { "body": { "chunks": [ - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_5wLPjysGUfkW1Qmf0atyinqy\",\"type\":\"function\",\"function\":{\"name\":\"json\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"amx81suv\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Ch7W4XbvpMt\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"rIUu2pigck\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"xNvztOfbO\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"PNy35Fz48\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"b65QDs7I36f\"}", - "data: {\"id\":\"chatcmpl-Dfl7qGo3YBgUa0NSGiLhgYJO1bJpF\",\"object\":\"chat.completion.chunk\",\"created\":1778844970,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_5b8b54db47\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"2Q\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_74H727j0x8ZHJToaW5IaF3eA\",\"type\":\"function\",\"function\":{\"name\":\"json\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Xxe1tPDc\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"etJeW2yGc0d\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"TS8pca2qtF\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"odo7j3dfx\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"qHcBMuPTp\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"B03JrfRjTH9\"}", + "data: {\"id\":\"chatcmpl-DgrwNq5jmFDM3XHsHrXDB05XrYaup\",\"object\":\"chat.completion.chunk\",\"created\":1779109495,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_54f26dc974\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"rd\"}", "data: [DONE]" ], "kind": "sse" @@ -1217,12 +1220,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d7e59c3fc2d4-VIE", + "cf-ray": "9fdb1207ff3134c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:36:10 GMT", + "date": "Mon, 18 May 2026 13:04:55 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "246", + "openai-processing-ms": "434", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1237,7 +1240,7 @@ "x-ratelimit-remaining-tokens": "149999990", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_980927ce0b5c4bba8c1ab923d005c164" + "x-request-id": "req_b6b51c6e67d647bb9f9a065713dd8b9d" }, "status": 200, "statusText": "OK" diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v5.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v5.cassette.json index 4322e8b0e..b4d8ac337 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v5.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v5.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "7c0e162d6794cf5d", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:03.240Z", + "recordedAt": "2026-05-18T13:05:39.458Z", "request": { "body": { "kind": "json", @@ -37,11 +37,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845023, - "created_at": 1778845022, + "completed_at": 1779109539, + "created_at": 1779109538, "error": null, "frequency_penalty": 0, - "id": "resp_0afb341d671c9e08006a07055e851c81949c93702050fe1145", + "id": "resp_01ed88a6e4d5e25f006a0b0ea283c4819798da88104fa47d5c", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -60,7 +60,7 @@ "type": "output_text" } ], - "id": "msg_0afb341d671c9e08006a07055f0ae08194bacc51eb2ec901cc", + "id": "msg_01ed88a6e4d5e25f006a0b0ea324088197b7aeb82953743981", "role": "assistant", "status": "completed", "type": "message" @@ -109,13 +109,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d92e1f9a5b0d-VIE", + "cf-ray": "9fdb1316fdb8d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:03 GMT", + "date": "Mon, 18 May 2026 13:05:39 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "723", + "openai-processing-ms": "946", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -129,7 +129,7 @@ "x-ratelimit-remaining-tokens": "149999962", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_6f8cd508a1b243a597a7c870142d940f" + "x-request-id": "req_8d234dbe098a4588981e873f967c8901" }, "status": 200, "statusText": "OK" @@ -139,7 +139,7 @@ "callIndex": 1, "id": "f12aff3913439248", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:04.063Z", + "recordedAt": "2026-05-18T13:05:41.931Z", "request": { "body": { "kind": "json", @@ -190,11 +190,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845023, - "created_at": 1778845023, + "completed_at": 1779109541, + "created_at": 1779109539, "error": null, "frequency_penalty": 0, - "id": "resp_07080156b4e9999d006a07055f74388196bdcbbc6d65f589a6", + "id": "resp_0125446c3b1fd902006a0b0ea3a774819690a2e087135a700b", "incomplete_details": null, "instructions": null, "max_output_tokens": 32, @@ -213,7 +213,7 @@ "type": "output_text" } ], - "id": "msg_07080156b4e9999d006a07055fcd608196aa4c775f2ae24186", + "id": "msg_0125446c3b1fd902006a0b0ea598988196ad4ff3736ffa2bfc", "role": "assistant", "status": "completed", "type": "message" @@ -275,13 +275,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d933dd815b0d-VIE", + "cf-ray": "9fdb131e2ab2d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:04 GMT", + "date": "Mon, 18 May 2026 13:05:41 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "611", + "openai-processing-ms": "2275", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -292,10 +292,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999937", + "x-ratelimit-remaining-tokens": "149999935", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_b8f6e1defb8b44bfa9fcf16b760290e4" + "x-request-id": "req_b8928cde60dd42d39bc8fb48f262d098" }, "status": 200, "statusText": "OK" @@ -305,7 +305,7 @@ "callIndex": 2, "id": "5d2312f425df9f2f", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:05.011Z", + "recordedAt": "2026-05-18T13:05:42.823Z", "request": { "body": { "kind": "json", @@ -334,20 +334,20 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0543c4689b3154e7006a07056044488194ae861ef845a5ec2f\",\"object\":\"response\",\"created_at\":1778845024,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0543c4689b3154e7006a07056044488194ae861ef845a5ec2f\",\"object\":\"response\",\"created_at\":1778845024,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"One\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"ebjPDPr9kqbnn\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"fbl2msFb8PPGfkx\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" two\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"Psrm022gc5vn\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"T8ATw93FNQx5VkE\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"Dgz5WqxCzK\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"obfuscation\":\"l8NFpXJMEwgst4R\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":10,\"text\":\"One, two, three.\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"},\"sequence_number\":11}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":12}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0543c4689b3154e7006a07056044488194ae861ef845a5ec2f\",\"object\":\"response\",\"created_at\":1778845024,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845024,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0543c4689b3154e7006a070560b60881949ab914142a87cf38\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":29},\"user\":null,\"metadata\":{}},\"sequence_number\":13}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0359c1fe2226b078006a0b0ea61c148196b2c9d6c65613f8e0\",\"object\":\"response\",\"created_at\":1779109542,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0359c1fe2226b078006a0b0ea61c148196b2c9d6c65613f8e0\",\"object\":\"response\",\"created_at\":1779109542,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"One\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"tlPmH6Zue9iaB\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"A17pOyjlUaUWS3D\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" two\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"uwASjrX6Zl8N\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"vLaK7tqHhclDRtt\",\"output_index\":0,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"A8bSZOAX4J\",\"output_index\":0,\"sequence_number\":8}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"obfuscation\":\"VxvEjC21LtGDwhf\",\"output_index\":0,\"sequence_number\":9}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":10,\"text\":\"One, two, three.\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"},\"sequence_number\":11}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":12}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0359c1fe2226b078006a0b0ea61c148196b2c9d6c65613f8e0\",\"object\":\"response\",\"created_at\":1779109542,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109542,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0359c1fe2226b078006a0b0ea67d6881968fc72a42501d95fe\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":29},\"user\":null,\"metadata\":{}},\"sequence_number\":13}" ], "kind": "sse" }, @@ -355,12 +355,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d938fada5b0d-VIE", + "cf-ray": "9fdb132d9e52d87b-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:37:04 GMT", + "date": "Mon, 18 May 2026 13:05:42 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "115", + "openai-processing-ms": "85", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -368,7 +368,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_438f4b631a1d4534abdeec8cbfb65bf2" + "x-request-id": "req_d19132582cd7441d9bb6500806bff4a7" }, "status": 200, "statusText": "OK" @@ -378,7 +378,7 @@ "callIndex": 0, "id": "7d3c0bb6426e6be4", "matchKey": "POST api.openai.com/v1/embeddings", - "recordedAt": "2026-05-15T11:37:05.292Z", + "recordedAt": "2026-05-18T13:05:43.003Z", "request": { "body": { "kind": "json", @@ -399,559 +399,556 @@ "data": [ { "embedding": [ - 0.0196380615234375, 0.019073486328125, -0.01320648193359375, - -0.0206451416015625, -0.0188751220703125, 0.02972412109375, - -0.020599365234375, 0.023651123046875, 0.006359100341796875, - -0.017059326171875, 0.004039764404296875, 0.01035308837890625, - -0.028961181640625, 0.00681304931640625, -0.03192138671875, - 0.02520751953125, -0.0282440185546875, 0.0501708984375, - 0.09149169921875, -0.020050048828125, 0.022308349609375, - -0.01296234130859375, -0.01445770263671875, 0.03436279296875, - 0.052764892578125, -0.0010938644409179688, 0.0081634521484375, - 0.042510986328125, 0.003047943115234375, 0.059844970703125, - 0.0164337158203125, -0.0078582763671875, - -0.0024394989013671875, 0.02423095703125, 0.0195159912109375, - -0.061065673828125, -0.0133514404296875, -0.038421630859375, - 0.00791168212890625, 0.027679443359375, 0.01708984375, - -0.0284271240234375, -0.0287322998046875, 0.03277587890625, - -0.035919189453125, -0.04705810546875, -0.0094451904296875, - -0.022491455078125, 0.02313232421875, -0.0037364959716796875, - 0.045654296875, -0.00487518310546875, 0.020172119140625, - 0.009246826171875, 0.00449371337890625, 0.022552490234375, - 0.03277587890625, 0.0433349609375, 0.02569580078125, - 0.0035457611083984375, 0.051788330078125, 0.020599365234375, - 0.04046630859375, 0.0177459716796875, -0.0055999755859375, - 0.03271484375, 0.0215606689453125, 0.032257080078125, - 0.050048828125, 0.0258026123046875, 0.018768310546875, - 0.044342041015625, -0.005954742431640625, 0.00675201416015625, - 0.0188751220703125, 0.01470947265625, 0.00463104248046875, - 0.038787841796875, 0.017364501953125, -0.03466796875, - -0.0016241073608398438, 0.01056671142578125, - 0.0137481689453125, -0.0216827392578125, -0.00678253173828125, - -0.01279449462890625, -0.00383758544921875, - 0.0027370452880859375, -0.0027408599853515625, - -0.0220947265625, 0.0014705657958984375, -0.0117034912109375, - -0.0172882080078125, 0.015655517578125, -0.03094482421875, - 0.00457000732421875, 0.0211029052734375, -0.04888916015625, - -0.058837890625, 0.02716064453125, 0.0279083251953125, - 0.0013456344604492188, -0.046783447265625, 0.0316162109375, - 0.0154876708984375, -0.01546478271484375, - 0.00037980079650878906, -0.0005192756652832031, - 0.0143280029296875, -0.040924072265625, -0.021728515625, - 0.004119873046875, -0.0002334117889404297, 0.032440185546875, - -0.059783935546875, 0.01092529296875, 0.014007568359375, - -0.00711822509765625, 0.031036376953125, - -0.007556915283203125, -0.00634002685546875, 0.0025634765625, - -0.04180908203125, 0.0404052734375, -0.03387451171875, - -0.048248291015625, 0.00656890869140625, 0.0265350341796875, - 0.0227813720703125, -0.00855255126953125, 0.00469970703125, - -0.044952392578125, 0.01690673828125, 0.025909423828125, - 0.05059814453125, -0.0096282958984375, 0.033782958984375, - -0.00574493408203125, 0.0175018310546875, 0.01776123046875, - -0.015625, 0.0285797119140625, -0.01055908203125, - 0.0364990234375, -0.018280029296875, -0.0338134765625, - -0.0313720703125, 0.040069580078125, 0.0101470947265625, - -0.04278564453125, 0.0164794921875, -0.0212554931640625, - 0.00272369384765625, 0.0289459228515625, - -0.003696441650390625, -0.0621337890625, 0.014434814453125, - -0.00196075439453125, 0.0384521484375, 0.025665283203125, - -0.036865234375, 0.0212860107421875, -0.046142578125, - -0.0162811279296875, -0.0034656524658203125, - 0.00821685791015625, -0.00901031494140625, - -0.00539398193359375, 0.011871337890625, 0.0160064697265625, - -0.0234527587890625, 0.004573822021484375, 0.0511474609375, - 0.038909912109375, 0.0225677490234375, -0.0139312744140625, - -0.018524169921875, 0.08013916015625, 0.026275634765625, - -0.0214385986328125, -0.01105499267578125, - -0.01430511474609375, 0.03076171875, -0.01468658447265625, - 0.0189056396484375, 0.0250244140625, -0.0206146240234375, - -0.0269317626953125, -0.0172882080078125, 0.05322265625, - 0.061492919921875, -0.0190887451171875, 0.056365966796875, - -0.04541015625, -0.0628662109375, 0.0241546630859375, - 0.002185821533203125, 0.0067596435546875, 0.00933074951171875, - 0.0079803466796875, -0.039215087890625, 0.041900634765625, - -0.0055999755859375, -0.0033016204833984375, - -0.01486968994140625, 0.019073486328125, -0.0100555419921875, - -0.04833984375, 0.029510498046875, -0.00475311279296875, - -0.01241302490234375, -0.005481719970703125, - -0.0262298583984375, 0.038421630859375, 0.038543701171875, - -0.01038360595703125, 0.012939453125, 0.033966064453125, - 0.0299072265625, -0.005496978759765625, 0.03466796875, - 0.023406982421875, 0.0113525390625, -0.0155792236328125, - -0.048919677734375, 0.055419921875, 0.05267333984375, - 0.00608062744140625, -0.04461669921875, 0.002704620361328125, - 0.00434112548828125, -0.0286407470703125, - -0.01476287841796875, -0.01540374755859375, - -0.0269317626953125, -0.019195556640625, 0.0035400390625, - -0.01326751708984375, -0.047821044921875, 0.0193634033203125, - -0.033843994140625, 0.0185394287109375, 0.0141448974609375, - 0.0026836395263671875, -0.00033974647521972656, - 0.002651214599609375, 0.01383209228515625, 0.03717041015625, - -0.042083740234375, 0.01270294189453125, 0.04339599609375, - -0.0172119140625, -0.025360107421875, 0.0172576904296875, - -0.035430908203125, 0.035125732421875, 0.00611114501953125, - -0.00634002685546875, -0.047698974609375, - -0.00983428955078125, 0.0104217529296875, 0.019866943359375, - -0.00997161865234375, -0.0018606185913085938, - -0.008392333984375, 0.019195556640625, -0.00960540771484375, - -0.023651123046875, -0.022491455078125, -0.030517578125, - 0.034912109375, -0.0008358955383300781, -0.0177154541015625, - 0.0290985107421875, -0.020965576171875, 0.005283355712890625, - -0.028228759765625, 0.0008840560913085938, -0.001129150390625, - 0.015777587890625, -0.0218505859375, 0.0443115234375, - -0.01230621337890625, 0.01255035400390625, - -0.00955963134765625, -0.01983642578125, 0.04046630859375, - 0.0672607421875, -0.0325927734375, 0.052032470703125, - 0.02166748046875, -0.021240234375, -0.033966064453125, - 0.022613525390625, -0.008087158203125, 0.006885528564453125, - 0.03558349609375, 0.0010814666748046875, 0.006816864013671875, - 0.0125274658203125, -0.0239105224609375, -0.01180267333984375, - 0.004608154296875, 0.0152740478515625, -0.04949951171875, - 0.0484619140625, -0.045501708984375, 0.019683837890625, - 0.0521240234375, 0.005802154541015625, -0.054107666015625, - -0.03643798828125, 0.04638671875, -0.0677490234375, - -0.01151275634765625, 0.007213592529296875, - -0.0158538818359375, 0.019561767578125, 0.03448486328125, - 0.011016845703125, -0.0061187744140625, 0.0125732421875, - 0.007537841796875, 0.002048492431640625, -0.0272369384765625, - 0.03173828125, 0.01447296142578125, 0.03839111328125, - -0.02783203125, -0.0063323974609375, 0.0008225440979003906, - 0.00875091552734375, -0.00853729248046875, 0.06134033203125, - -0.0401611328125, -0.021484375, 0.0031414031982421875, - 0.029296875, 0.003040313720703125, -0.0085601806640625, - -0.0253448486328125, 0.0270233154296875, -0.035919189453125, - 0.0270233154296875, -0.0033855438232421875, - -0.01428985595703125, 0.0377197265625, 0.0011196136474609375, - -0.00563812255859375, -0.03594970703125, -0.0191650390625, - -0.0196075439453125, 0.0004887580871582031, 0.0313720703125, - -0.040771484375, 0.0192108154296875, 0.064208984375, - 0.0094451904296875, -0.05694580078125, 0.01303863525390625, - -0.0164947509765625, -0.052764892578125, -0.0163116455078125, - -0.018951416015625, -0.036712646484375, -0.0084228515625, - -0.0016994476318359375, -0.005573272705078125, + 0.0196075439453125, 0.019073486328125, -0.01314544677734375, + -0.020660400390625, -0.0188140869140625, 0.029632568359375, + -0.0206298828125, 0.0236358642578125, 0.006320953369140625, + -0.01708984375, 0.004039764404296875, 0.0103607177734375, + -0.0289459228515625, 0.0068359375, -0.03192138671875, + 0.0251922607421875, -0.0282745361328125, 0.050262451171875, + 0.0914306640625, -0.02008056640625, 0.0222930908203125, + -0.01297760009765625, -0.0144805908203125, 0.034393310546875, + 0.052734375, -0.0011014938354492188, 0.00817108154296875, + 0.042510986328125, 0.0030460357666015625, 0.05987548828125, + 0.0164642333984375, -0.00787353515625, -0.00244903564453125, + 0.02423095703125, 0.01953125, -0.061065673828125, + -0.013336181640625, -0.03851318359375, 0.0079498291015625, + 0.0276641845703125, 0.0171051025390625, -0.02838134765625, + -0.0286865234375, 0.03277587890625, -0.035919189453125, + -0.047088623046875, -0.00940704345703125, -0.0224761962890625, + 0.02313232421875, -0.0037555694580078125, 0.045654296875, + -0.004848480224609375, 0.02020263671875, 0.00923919677734375, + 0.0044708251953125, 0.02252197265625, 0.03271484375, + 0.0433349609375, 0.0257110595703125, 0.0035400390625, + 0.051788330078125, 0.0206298828125, 0.04046630859375, + 0.01776123046875, -0.00559234619140625, 0.03271484375, + 0.021575927734375, 0.0322265625, 0.050048828125, + 0.02581787109375, 0.0187835693359375, 0.044342041015625, + -0.005947113037109375, 0.006748199462890625, 0.01885986328125, + 0.014678955078125, 0.004638671875, 0.038848876953125, + 0.0173492431640625, -0.03466796875, -0.0016269683837890625, + 0.01055908203125, 0.01372528076171875, -0.02166748046875, + -0.00677490234375, -0.01284027099609375, -0.00388336181640625, + 0.00274658203125, -0.002719879150390625, -0.0220794677734375, + 0.0014619827270507812, -0.01175689697265625, -0.0172119140625, + 0.015655517578125, -0.0308990478515625, 0.00458526611328125, + 0.0211334228515625, -0.04888916015625, -0.058837890625, + 0.02716064453125, 0.027862548828125, 0.001308441162109375, + -0.04681396484375, 0.0316162109375, 0.0155029296875, + -0.01544952392578125, 0.0004296302795410156, + -0.0005507469177246094, 0.01428985595703125, -0.0408935546875, + -0.021759033203125, 0.00412750244140625, -0.00025177001953125, + 0.032440185546875, -0.059783935546875, 0.01093292236328125, + 0.01397705078125, -0.00713348388671875, 0.031005859375, + -0.007572174072265625, -0.006343841552734375, + 0.0026073455810546875, -0.04180908203125, 0.040374755859375, + -0.03387451171875, -0.048248291015625, 0.006519317626953125, + 0.0264739990234375, 0.0227203369140625, -0.00856781005859375, + 0.004657745361328125, -0.044952392578125, 0.016876220703125, + 0.0258941650390625, 0.0506591796875, -0.00958251953125, + 0.033721923828125, -0.0057525634765625, 0.01751708984375, + 0.0177154541015625, -0.01558685302734375, 0.028533935546875, + -0.01058197021484375, 0.036529541015625, -0.0183258056640625, + -0.033843994140625, -0.031402587890625, 0.040069580078125, + 0.01015472412109375, -0.04278564453125, 0.016510009765625, + -0.0212554931640625, 0.00275421142578125, 0.0289459228515625, + -0.0037078857421875, -0.062103271484375, 0.01442718505859375, + -0.0018835067749023438, 0.0384521484375, 0.02569580078125, + -0.036865234375, 0.021331787109375, -0.046173095703125, + -0.016265869140625, -0.0034809112548828125, + 0.0081939697265625, -0.0090179443359375, -0.00542449951171875, + 0.011871337890625, 0.0159454345703125, -0.0235137939453125, + 0.00461578369140625, 0.051116943359375, 0.0389404296875, + 0.0225677490234375, -0.01392364501953125, -0.0185699462890625, + 0.08013916015625, 0.0262603759765625, -0.02142333984375, + -0.01103973388671875, -0.01428985595703125, 0.030731201171875, + -0.01473236083984375, 0.0189361572265625, 0.0250244140625, + -0.0206451416015625, -0.026885986328125, -0.0172882080078125, + 0.05322265625, 0.0615234375, -0.019073486328125, + 0.056396484375, -0.045440673828125, -0.06280517578125, + 0.0241546630859375, 0.0021800994873046875, + 0.006801605224609375, 0.0093231201171875, 0.00798797607421875, + -0.03924560546875, 0.04193115234375, -0.0056304931640625, + -0.00327301025390625, -0.01485443115234375, + 0.0191192626953125, -0.01004791259765625, -0.048309326171875, + 0.0295257568359375, -0.004802703857421875, + -0.0124359130859375, -0.00547027587890625, + -0.0262298583984375, 0.0384521484375, 0.03857421875, + -0.01038360595703125, 0.012908935546875, 0.034027099609375, + 0.0299072265625, -0.00550079345703125, 0.034637451171875, + 0.023406982421875, 0.0113067626953125, -0.01557159423828125, + -0.048919677734375, 0.055419921875, 0.052642822265625, + 0.00609588623046875, -0.044586181640625, 0.002712249755859375, + 0.00432586669921875, -0.028656005859375, -0.0147552490234375, + -0.015411376953125, -0.0269775390625, -0.01922607421875, + 0.003505706787109375, -0.0132598876953125, -0.04779052734375, + 0.019317626953125, -0.0338134765625, 0.01849365234375, + 0.01416015625, 0.0026702880859375, -0.00037384033203125, + 0.0026149749755859375, 0.01383209228515625, 0.037200927734375, + -0.0421142578125, 0.0126953125, 0.043365478515625, + -0.0171966552734375, -0.025360107421875, 0.017303466796875, + -0.03546142578125, 0.035125732421875, 0.006137847900390625, + -0.00630950927734375, -0.047698974609375, -0.0099029541015625, + 0.01043701171875, 0.0198974609375, -0.00995635986328125, + -0.0018243789672851562, -0.00839996337890625, + 0.0192108154296875, -0.0095977783203125, -0.023651123046875, + -0.0224609375, -0.030517578125, 0.03485107421875, + -0.0008287429809570312, -0.0177154541015625, 0.029052734375, + -0.020965576171875, 0.005268096923828125, -0.028167724609375, + 0.0009093284606933594, -0.001140594482421875, + 0.0157928466796875, -0.02178955078125, 0.0443115234375, + -0.01227569580078125, 0.01256561279296875, + -0.0095367431640625, -0.0198211669921875, 0.04046630859375, + 0.06732177734375, -0.032562255859375, 0.052032470703125, + 0.02166748046875, -0.021209716796875, -0.033935546875, + 0.0225982666015625, -0.008056640625, 0.006862640380859375, + 0.035552978515625, 0.0010929107666015625, + 0.006832122802734375, 0.012481689453125, -0.0239105224609375, + -0.0117950439453125, 0.004608154296875, 0.01525115966796875, + -0.049560546875, 0.048431396484375, -0.045501708984375, + 0.0196685791015625, 0.0521240234375, 0.00585174560546875, + -0.054107666015625, -0.036376953125, 0.04632568359375, + -0.06768798828125, -0.01151275634765625, 0.00713348388671875, + -0.0157928466796875, 0.0195770263671875, 0.03448486328125, + 0.01102447509765625, -0.00614166259765625, + 0.01256561279296875, 0.007549285888671875, + 0.0020580291748046875, -0.0272674560546875, 0.031707763671875, + 0.0144805908203125, 0.03839111328125, -0.0278778076171875, + -0.00634765625, 0.0008325576782226562, 0.00878143310546875, + -0.008514404296875, 0.06134033203125, -0.040191650390625, + -0.021484375, 0.003170013427734375, 0.029327392578125, + 0.003055572509765625, -0.00859832763671875, -0.02532958984375, + 0.0270538330078125, -0.035919189453125, 0.0270233154296875, + -0.0033893585205078125, -0.01428985595703125, 0.0377197265625, + 0.001129150390625, -0.005687713623046875, -0.03594970703125, + -0.0191802978515625, -0.0195770263671875, + 0.0004911422729492188, 0.0313720703125, -0.040771484375, + 0.0191802978515625, 0.06414794921875, 0.00946807861328125, + -0.05694580078125, 0.0130462646484375, -0.016510009765625, + -0.052764892578125, -0.0163116455078125, -0.018951416015625, + -0.03668212890625, -0.0084381103515625, + -0.0017042160034179688, -0.00557708740234375, -0.03851318359375, 0.07061767578125, -0.049896240234375, - -0.027740478515625, 0.0287628173828125, -0.028839111328125, - 0.04705810546875, -0.0303955078125, -0.0250396728515625, - -0.008270263671875, -0.047637939453125, -0.0262298583984375, - -0.0643310546875, -0.0024242401123046875, 0.0355224609375, - -0.047149658203125, 0.032012939453125, 0.03704833984375, - 0.043701171875, -0.05511474609375, 0.01226806640625, - -0.006153106689453125, 0.012237548828125, 0.00476837158203125, - 0.00540924072265625, 0.02252197265625, -0.03302001953125, - -0.0009093284606933594, -0.0104217529296875, - -0.00444793701171875, -0.0295257568359375, - -0.0172882080078125, 0.002864837646484375, -0.039031982421875, - 0.01010894775390625, -0.006862640380859375, 0.042938232421875, - 0.028167724609375, 0.007511138916015625, -0.0171661376953125, - 0.0216217041015625, -0.022247314453125, 0.00659942626953125, - -0.03790283203125, 0.01537322998046875, - -0.0037822723388671875, 0.01319122314453125, - 0.054046630859375, 0.0298919677734375, -0.07373046875, - 0.033111572265625, -0.021514892578125, 0.03753662109375, - 0.004749298095703125, -0.058837890625, 0.01134490966796875, - 0.0028514862060546875, -0.007328033447265625, - -0.01137542724609375, -0.06524658203125, -0.0193023681640625, - 0.03265380859375, 0.00508880615234375, 0.0213623046875, - 0.0005936622619628906, -0.0084991455078125, - 0.0110015869140625, 0.01507568359375, 0.0099639892578125, - 0.01019287109375, -0.01430511474609375, 0.0016298294067382812, - 0.047576904296875, -0.017364501953125, 0.01197052001953125, - -0.03570556640625, -0.0462646484375, -0.0174102783203125, - 0.055755615234375, 0.0299072265625, -0.045867919921875, - 0.060028076171875, -0.031219482421875, -0.002826690673828125, - -0.0088043212890625, -0.01319122314453125, - -0.0246429443359375, 0.0300750732421875, 0.0197296142578125, - 0.06988525390625, 0.00811767578125, -0.03424072265625, - 0.004909515380859375, 0.046600341796875, - -0.0038928985595703125, 0.044769287109375, 0.032745361328125, - 0.018707275390625, 0.024261474609375, 0.0193939208984375, - -0.038818359375, -0.031280517578125, 0.015869140625, - 0.0335693359375, 0.0224456787109375, 0.02581787109375, - 0.007049560546875, 0.00487518310546875, -0.059600830078125, - 0.03875732421875, -0.0494384765625, -0.06390380859375, - -0.0011310577392578125, 0.021942138671875, - -0.0187530517578125, 0.0063018798828125, -0.0794677734375, - -0.00656890869140625, 0.00356292724609375, - 0.0019741058349609375, -0.0234832763671875, - 0.0159759521484375, 0.052032470703125, -0.038421630859375, - 0.01377105712890625, -0.00019228458404541016, - 0.0215606689453125, -0.006824493408203125, -0.053955078125, - -0.0105743408203125, -0.0272216796875, -0.02362060546875, - 0.023193359375, 0.0016756057739257812, 0.0499267578125, - 0.0129547119140625, 0.033172607421875, 0.08331298828125, - 0.0103759765625, 0.003932952880859375, 0.00826263427734375, - -0.0260772705078125, 0.0300445556640625, -0.0252685546875, - 0.0322265625, 0.01165008544921875, -0.01071929931640625, - -0.0037631988525390625, -0.02313232421875, 0.0157470703125, - 0.0116424560546875, 0.0026683807373046875, 0.0199127197265625, - 0.0246124267578125, -0.01528167724609375, 0.004302978515625, - 0.032928466796875, 0.01409149169921875, -0.024322509765625, - -0.007480621337890625, 0.04766845703125, -0.04901123046875, - 0.0227203369140625, -0.0255889892578125, -0.014129638671875, - -0.0164337158203125, 0.0095062255859375, 0.01837158203125, - 0.01837158203125, 0.05340576171875, 0.00007402896881103516, - 0.03070068359375, 0.003925323486328125, -0.0161285400390625, - -0.00675201416015625, -0.0189666748046875, - -0.01511383056640625, -0.0167999267578125, 0.0219573974609375, - -0.0273590087890625, -0.034515380859375, -0.01702880859375, + -0.0277252197265625, 0.0287322998046875, -0.02880859375, + 0.047088623046875, -0.030364990234375, -0.025054931640625, + -0.008270263671875, -0.04766845703125, -0.02618408203125, + -0.0643310546875, -0.002410888671875, 0.0355224609375, + -0.04705810546875, 0.03204345703125, 0.0369873046875, + 0.043701171875, -0.0550537109375, 0.01226043701171875, + -0.006168365478515625, 0.012237548828125, + 0.004772186279296875, 0.00537872314453125, 0.022552490234375, + -0.03302001953125, -0.0009675025939941406, -0.0103759765625, + -0.00447845458984375, -0.029510498046875, -0.0172576904296875, + 0.0028629302978515625, -0.0390625, 0.01015472412109375, + -0.006866455078125, 0.04296875, 0.028167724609375, + 0.0075225830078125, -0.01715087890625, 0.0215911865234375, + -0.0222625732421875, 0.006572723388671875, -0.037872314453125, + 0.015380859375, -0.003780364990234375, 0.0131683349609375, + 0.0540771484375, 0.0298919677734375, -0.07373046875, + 0.0330810546875, -0.02154541015625, 0.037506103515625, + 0.004802703857421875, -0.058837890625, 0.0113525390625, + 0.0028858184814453125, -0.007354736328125, + -0.01139068603515625, -0.06524658203125, -0.019317626953125, + 0.032623291015625, 0.00508880615234375, 0.021331787109375, + 0.0006260871887207031, -0.008514404296875, + 0.01102447509765625, 0.0150909423828125, 0.0099639892578125, + 0.01019287109375, -0.01428985595703125, 0.00167083740234375, + 0.047515869140625, -0.017364501953125, 0.01197052001953125, + -0.03564453125, -0.0462646484375, -0.017425537109375, + 0.055816650390625, 0.029937744140625, -0.045867919921875, + 0.06005859375, -0.0311737060546875, -0.00281524658203125, + -0.00878143310546875, -0.01318359375, -0.0246124267578125, + 0.0300140380859375, 0.0197296142578125, 0.0699462890625, + 0.00809478759765625, -0.03424072265625, 0.00490570068359375, + 0.046600341796875, -0.00392913818359375, 0.044769287109375, + 0.03271484375, 0.018768310546875, 0.0243072509765625, + 0.019378662109375, -0.038787841796875, -0.03131103515625, + 0.0158538818359375, 0.0335693359375, 0.0224456787109375, + 0.025848388671875, 0.007061004638671875, 0.0049285888671875, + -0.0595703125, 0.038726806640625, -0.0494384765625, + -0.06390380859375, -0.0011005401611328125, 0.0219268798828125, + -0.018798828125, 0.006317138671875, -0.0794677734375, + -0.00662994384765625, 0.00353240966796875, + 0.0020122528076171875, -0.0234527587890625, + 0.0159759521484375, 0.052032470703125, -0.038360595703125, + 0.01372528076171875, -0.00020372867584228516, + 0.021575927734375, -0.00679779052734375, -0.053985595703125, + -0.01056671142578125, -0.0272369384765625, + -0.0236053466796875, 0.023162841796875, 0.00165557861328125, + 0.0499267578125, 0.012908935546875, 0.033233642578125, + 0.08331298828125, 0.01042938232421875, 0.0039825439453125, + 0.00824737548828125, -0.0260772705078125, 0.0299530029296875, + -0.025299072265625, 0.032196044921875, 0.01166534423828125, + -0.010711669921875, -0.00376129150390625, -0.023162841796875, + 0.0157470703125, 0.01163482666015625, 0.002651214599609375, + 0.0198822021484375, 0.0246124267578125, -0.0152740478515625, + 0.004291534423828125, 0.032928466796875, 0.0141143798828125, + -0.024322509765625, -0.007518768310546875, 0.04766845703125, + -0.04901123046875, 0.0227508544921875, -0.0255584716796875, + -0.0141143798828125, -0.01641845703125, 0.0095367431640625, + 0.0183563232421875, 0.0184173583984375, 0.05340576171875, + 0.00008529424667358398, 0.03070068359375, + 0.0038967132568359375, -0.0161285400390625, + -0.006809234619140625, -0.0189361572265625, + -0.01512908935546875, -0.0167999267578125, 0.022003173828125, + -0.027374267578125, -0.03448486328125, -0.016998291015625, -0.024627685546875, 0.045806884765625, -0.03558349609375, - -0.0439453125, -0.0007295608520507812, -0.0037212371826171875, - -0.00934600830078125, -0.00396728515625, 0.0269317626953125, - -0.01520538330078125, -0.00968170166015625, 0.063232421875, - 0.0379638671875, 0.0172882080078125, 0.01885986328125, - 0.036865234375, 0.00424957275390625, 0.010406494140625, - -0.01483917236328125, -0.022186279296875, - -0.01180267333984375, 0.0484619140625, 0.01297760009765625, - 0.0174713134765625, 0.020751953125, 0.01824951171875, - 0.01013946533203125, -0.00399017333984375, -0.029998779296875, - 0.0654296875, -0.005413055419921875, 0.0135650634765625, - 0.032470703125, -0.03521728515625, -0.04522705078125, - 0.0218505859375, 0.003498077392578125, -0.002376556396484375, - -0.0419921875, -0.0294647216796875, -0.003753662109375, - 0.01380157470703125, 0.0174560546875, -0.0234375, - 0.01435089111328125, 0.0014057159423828125, - 0.0113372802734375, 0.000843048095703125, -0.035858154296875, - -0.01200103759765625, 0.0024013519287109375, - 0.0251007080078125, -0.0193023681640625, -0.0242462158203125, - 0.01535797119140625, -0.0369873046875, 0.00672149658203125, - -0.0191650390625, 0.022369384765625, -0.0075225830078125, - 0.0176544189453125, -0.0187530517578125, 0.0154266357421875, - 0.028350830078125, -0.00547027587890625, 0.0279541015625, - -0.0240631103515625, 0.02398681640625, -0.0128021240234375, - -0.007293701171875, 0.0142974853515625, 0.03460693359375, - -0.032012939453125, 0.005584716796875, -0.00534820556640625, - 0.04705810546875, 0.018890380859375, 0.0184326171875, - -0.01000213623046875, -0.0172882080078125, 0.0244140625, - 0.01416778564453125, 0.045562744140625, 0.012786865234375, - -0.0233917236328125, -0.025146484375, -0.00409698486328125, - -0.019775390625, 0.00835418701171875, -0.0301971435546875, - 0.041046142578125, 0.012481689453125, -0.0015058517456054688, - 0.0059967041015625, -0.00667572021484375, 0.01248931884765625, + -0.0439453125, -0.000728607177734375, -0.0036830902099609375, + -0.009368896484375, -0.0039520263671875, 0.026947021484375, + -0.01522064208984375, -0.00968170166015625, 0.06329345703125, + 0.037994384765625, 0.017242431640625, 0.0188751220703125, + 0.036865234375, 0.00426483154296875, 0.01039886474609375, + -0.01483154296875, -0.022216796875, -0.0118408203125, + 0.0484619140625, 0.01296234130859375, 0.0175018310546875, + 0.02081298828125, 0.0182037353515625, 0.010162353515625, + -0.003997802734375, -0.0299224853515625, 0.0654296875, + -0.0054168701171875, 0.01355743408203125, 0.032470703125, + -0.03521728515625, -0.04522705078125, 0.0218353271484375, + 0.0035457611083984375, -0.002376556396484375, -0.0419921875, + -0.0294342041015625, -0.0037860870361328125, + 0.01380157470703125, 0.0174713134765625, -0.0234527587890625, + 0.01428985595703125, 0.0014009475708007812, 0.01141357421875, + 0.0008606910705566406, -0.035858154296875, + -0.01201629638671875, 0.00235748291015625, 0.0251312255859375, + -0.019287109375, -0.0242156982421875, 0.01537322998046875, + -0.036956787109375, 0.00670623779296875, -0.0191650390625, + 0.0223846435546875, -0.007534027099609375, 0.0176849365234375, + -0.0187530517578125, 0.01541900634765625, 0.028289794921875, + -0.005496978759765625, 0.02801513671875, -0.02410888671875, + 0.023956298828125, -0.01284027099609375, -0.00730133056640625, + 0.01430511474609375, 0.03460693359375, -0.031982421875, + 0.0055694580078125, -0.005382537841796875, 0.047088623046875, + 0.0189361572265625, 0.0183868408203125, -0.0099945068359375, + -0.0172576904296875, 0.024444580078125, 0.0141754150390625, + 0.045562744140625, 0.0128021240234375, -0.0233917236328125, + -0.0251007080078125, -0.004085540771484375, + -0.0197906494140625, 0.008392333984375, -0.03021240234375, + 0.041046142578125, 0.0124664306640625, -0.0015897750854492188, + 0.0059967041015625, -0.006702423095703125, 0.0124664306640625, 0.02105712890625, 0.04742431640625, 0.0295562744140625, - -0.04473876953125, 0.05364990234375, 0.021392822265625, - -0.01189422607421875, 0.003597259521484375, - 0.004817962646484375, 0.03076171875, -0.0034389495849609375, - 0.0184326171875, -0.0283966064453125, 0.05914306640625, - -0.0222625732421875, -0.0287017822265625, 0.028778076171875, - -0.0350341796875, -0.0222015380859375, 0.02850341796875, - 0.022308349609375, 0.01444244384765625, 0.04388427734375, - 0.028167724609375, -0.04718017578125, -0.03582763671875, - 0.04150390625, 0.003902435302734375, 0.0093536376953125, - -0.02203369140625, -0.004726409912109375, -0.0140228271484375, - -0.049285888671875, -0.0005688667297363281, -0.006591796875, - 0.0171356201171875, -0.01517486572265625, 0.0199432373046875, - -0.013031005859375, -0.0202789306640625, - -0.002544403076171875, 0.005962371826171875, 0.0185546875, - -0.01016998291015625, -0.021697998046875, - -0.01117706298828125, -0.0189361572265625, 0.0223541259765625, - 0.0265960693359375, -0.0296173095703125, 0.019287109375, - -0.0141754150390625, 0.01216888427734375, 0.034698486328125, - -0.033782958984375, 0.0007343292236328125, -0.034149169921875, - 0.0001068115234375, 0.0009908676147460938, 0.0228271484375, - 0.00920867919921875, -0.01145172119140625, - -0.004093170166015625, 0.00984954833984375, - -0.0168609619140625, -0.039886474609375, - -0.007373809814453125, 0.007198333740234375, - 0.025665283203125, 0.01123046875, 0.0092315673828125, - -0.03607177734375, 0.0251922607421875, -0.01788330078125, - -0.007472991943359375, -0.03778076171875, -0.0200042724609375, - -0.01473236083984375, -0.01132965087890625, 0.0115966796875, - -0.0003600120544433594, -0.014190673828125, - -0.0032367706298828125, 0.0017652511596679688, - -0.023895263671875, -0.02532958984375, -0.0113525390625, - 0.0139923095703125, 0.01377105712890625, 0.01204681396484375, - 0.00659942626953125, -0.0007576942443847656, - -0.05926513671875, 0.007549285888671875, -0.033905029296875, - 0.00885009765625, 0.00292205810546875, -0.023193359375, - 0.022186279296875, 0.06939697265625, -0.02947998046875, - -0.0032253265380859375, -0.009796142578125, - -0.0232391357421875, -0.0031147003173828125, - -0.0014448165893554688, 0.0133056640625, - -0.003246307373046875, 0.033447265625, -0.0108184814453125, - 0.02032470703125, 0.00016582012176513672, - -0.0002765655517578125, -0.0404052734375, 0.00383758544921875, - 0.01140594482421875, -0.025543212890625, - -0.005756378173828125, 0.0184478759765625, -0.027191162109375, - 0.010162353515625, -0.01020050048828125, -0.0146636962890625, - -0.01611328125, -0.0227813720703125, 0.0262298583984375, - -0.036590576171875, -0.0228118896484375, 0.01200103759765625, - 0.03369140625, -0.04632568359375, -0.012908935546875, - -0.033050537109375, -0.035858154296875, 0.001895904541015625, - -0.00644683837890625, 0.0162811279296875, 0.0193023681640625, - -0.00662994384765625, -0.01361846923828125, 0.04168701171875, - -0.0478515625, 0.01113128662109375, -0.018890380859375, - 0.00014603137969970703, -0.015716552734375, - -0.01471710205078125, -0.01258087158203125, 0.00994873046875, - -0.028350830078125, 0.05316162109375, -0.02044677734375, - -0.0269927978515625, -0.0506591796875, -0.0142364501953125, - 0.0009512901306152344, 0.00801849365234375, - 0.0181732177734375, 0.04010009765625, 0.01983642578125, - -0.028778076171875, -0.004695892333984375, - 0.0011377334594726562, 0.0002841949462890625, - -0.007450103759765625, -0.038360595703125, - -0.01195526123046875, -0.037689208984375, 0.03973388671875, - -0.0227813720703125, -0.0092620849609375, -0.038116455078125, - 0.0167388916015625, 0.01529693603515625, - -0.002117156982421875, -0.01233673095703125, - -0.00740814208984375, -0.04132080078125, 0.03277587890625, - -0.026519775390625, 0.033172607421875, 0.0306549072265625, - -0.0022640228271484375, 0.0081024169921875, - -0.0269317626953125, 0.020233154296875, 0.0038509368896484375, - 0.0226898193359375, 0.00959014892578125, 0.039459228515625, - 0.003368377685546875, -0.0233306884765625, 0.0083160400390625, - 0.01971435546875, -0.035430908203125, 0.0504150390625, - 0.006153106689453125, -0.05767822265625, 0.00940704345703125, - -0.005260467529296875, 0.01422882080078125, - -0.0123748779296875, -0.00731658935546875, - 0.006916046142578125, -0.0230865478515625, 0.05035400390625, - -0.0291595458984375, 0.022705078125, -0.03564453125, - 0.039642333984375, 0.004154205322265625, 0.03436279296875, - 0.0035457611083984375, -0.0321044921875, -0.025146484375, - 0.003063201904296875, -0.030975341796875, -0.033599853515625, - 0.0011835098266601562, -0.00994873046875, -0.018798828125, - -0.03802490234375, -0.0288848876953125, 0.05987548828125, - 0.005603790283203125, -0.032440185546875, - 0.0026149749755859375, 0.011993408203125, - 0.001346588134765625, 0.0190887451171875, -0.009613037109375, - 0.01007080078125, -0.0171356201171875, -0.0546875, - -0.007312774658203125, 0.0242767333984375, 0.04388427734375, - -0.00441741943359375, -0.030609130859375, -0.0135498046875, - -0.0357666015625, -0.0108795166015625, 0.0157928466796875, - 0.0311126708984375, 0.01108551025390625, - 0.0009121894836425781, 0.01349639892578125, - -0.005985260009765625, -0.031585693359375, 0.019256591796875, - -0.029876708984375, 0.02435302734375, -0.0038127899169921875, - -0.036712646484375, -0.033660888671875, 0.0157012939453125, - 0.027618408203125, -0.0020122528076171875, 0.0290679931640625, - 0.01519775390625, -0.025604248046875, -0.01172637939453125, - -0.003452301025390625, -0.0303955078125, -0.00453948974609375, - -0.027496337890625, -0.01751708984375, 0.0164794921875, - 0.049224853515625, -0.044769287109375, 0.0099639892578125, - 0.03118896484375, -0.033203125, 0.0185546875, - -0.03277587890625, 0.02734375, -0.038726806640625, - -0.053253173828125, 0.024932861328125, 0.00039267539978027344, - -0.016571044921875, -0.0008702278137207031, - 0.006275177001953125, -0.021728515625, -0.004669189453125, - 0.031402587890625, -0.0042877197265625, -0.0088653564453125, - -0.010894775390625, 0.032623291015625, -0.046173095703125, - -0.0183563232421875, -0.0092620849609375, 0.040618896484375, - 0.006977081298828125, 0.0242156982421875, - 0.007274627685546875, 0.0134124755859375, 0.00235748291015625, - 0.00870513916015625, 0.0201873779296875, -0.033050537109375, - 0.05487060546875, 0.0231170654296875, -0.0002980232238769531, - -0.050750732421875, -0.0233917236328125, -0.01499176025390625, - 0.02587890625, 0.00461578369140625, -0.01174163818359375, - 0.026611328125, -0.0006389617919921875, -0.002239227294921875, - -0.0010595321655273438, -0.01160430908203125, - 0.008819580078125, 0.01372528076171875, -0.01226806640625, - 0.002239227294921875, -0.0183258056640625, 0.0221405029296875, - 0.0283203125, 0.0421142578125, -0.00855255126953125, - -0.034881591796875, 0.0240020751953125, 0.0377197265625, - -0.0155029296875, -0.03912353515625, 0.055145263671875, - 0.011260986328125, 0.004222869873046875, - 0.0019702911376953125, -0.0081024169921875, - 0.0102691650390625, -0.0025806427001953125, - -0.0022258758544921875, -0.00988006591796875, - -0.013153076171875, -0.035858154296875, 0.0132598876953125, - -0.0107879638671875, 0.019256591796875, 0.009033203125, - 0.00010395050048828125, 0.01342010498046875, -0.00390625, - 0.0177001953125, 0.028350830078125, 0.0125732421875, - 0.00830841064453125, 0.04632568359375, -0.054901123046875, - -0.051239013671875, 0.031005859375, 0.02239990234375, - 0.0031948089599609375, -0.01383209228515625, -0.0230712890625, - -0.00870513916015625, 0.03656005859375, -0.01050567626953125, - -0.031768798828125, -0.00438690185546875, -0.033843994140625, - -0.00930023193359375, 0.00926971435546875, -0.0279541015625, - 0.01134490966796875, 0.02227783203125, 0.017974853515625, - -0.006488800048828125, -0.0110931396484375, - -0.005939483642578125, 0.005809783935546875, -0.0125732421875, - -0.0092620849609375, -0.0013303756713867188, - -0.0197906494140625, -0.005931854248046875, -0.02630615234375, - -0.01263427734375, -0.006008148193359375, 0.017303466796875, - -0.0138702392578125, -0.0419921875, 0.0295562744140625, - -0.015716552734375, 0.0291748046875, 0.0355224609375, - 0.00782012939453125, -0.013824462890625, - -0.0011453628540039062, -0.009033203125, - -0.0012044906616210938, 0.002529144287109375, - 0.0003771781921386719, 0.01983642578125, -0.0207061767578125, - 0.01340484619140625, 0.003055572509765625, 0.031707763671875, - -0.0019130706787109375, -0.05224609375, -0.023223876953125, - -0.012176513671875, -0.01666259765625, -0.040008544921875, - -0.041229248046875, -0.01800537109375, -0.0038776397705078125, - 0.005023956298828125, -0.0029544830322265625, - 0.03387451171875, -0.011199951171875, 0.0279388427734375, - -0.01776123046875, 0.0246124267578125, 0.032867431640625, - 0.0034275054931640625, 0.0240325927734375, -0.014434814453125, - 0.040771484375, -0.0123138427734375, 0.006633758544921875, - -0.00806427001953125, 0.006805419921875, -0.00426483154296875, - 0.0221405029296875, -0.024749755859375, 0.040069580078125, - 0.013824462890625, 0.0361328125, -0.0028667449951171875, - -0.0037746429443359375, 0.06475830078125, 0.030303955078125, - -0.043304443359375, -0.04559326171875, 0.0207366943359375, - -0.0275115966796875, -0.0256805419921875, -0.01324462890625, - 0.004016876220703125, 0.050048828125, -0.01021575927734375, - -0.0172119140625, 0.048004150390625, 0.0139617919921875, - -0.0207061767578125, -0.0168914794921875, 0.0146026611328125, - 0.0286407470703125, 0.037322998046875, -0.0244598388671875, - 0.006633758544921875, -0.0207366943359375, 0.03302001953125, - 0.005397796630859375, -0.0284423828125, 0.0184783935546875, - -0.01146697998046875, 0.0014848709106445312, - -0.005641937255859375, 0.03778076171875, 0.0264434814453125, - 0.0123748779296875, -0.01178741455078125, 0.03521728515625, - 0.013092041015625, 0.01140594482421875, 0.00672149658203125, - 0.024505615234375, 0.019775390625, -0.0177154541015625, - -0.0242156982421875, 0.02166748046875, -0.0004470348358154297, - 0.00811004638671875, -0.013153076171875, - 0.0012683868408203125, -0.01226806640625, - 0.003078460693359375, 0.041107177734375, 0.0124969482421875, - -0.030029296875, -0.0092620849609375, -0.01904296875, - 0.00907135009765625, -0.0301055908203125, - -0.007450103759765625, -0.01367950439453125, - 0.0297698974609375, 0.032867431640625, -0.0117034912109375, - 0.0015239715576171875, 0.045867919921875, 0.00350189208984375, - -0.034942626953125, 0.023406982421875, 0.03289794921875, - -0.0280914306640625, 0.018035888671875, 0.010223388671875, - -0.00412750244140625, -0.003978729248046875, - -0.024566650390625, -0.01024627685546875, -0.031585693359375, - -0.05474853515625, 0.016815185546875, 0.0265045166015625, - 0.02679443359375, -0.01230621337890625, -0.014892578125, - 0.02044677734375, 0.0153656005859375, -0.06256103515625, - -0.0162506103515625, 0.0221099853515625, -0.017120361328125, - -0.015594482421875, -0.035980224609375, 0.0192718505859375, + -0.04473876953125, 0.053680419921875, 0.0214385986328125, + -0.01190185546875, 0.003604888916015625, 0.004787445068359375, + 0.03082275390625, -0.00344085693359375, 0.0184173583984375, + -0.0284271240234375, 0.05914306640625, -0.0222625732421875, + -0.0287017822265625, 0.02880859375, -0.035003662109375, + -0.022186279296875, 0.0284881591796875, 0.0223388671875, + 0.0144500732421875, 0.043853759765625, 0.0281829833984375, + -0.047088623046875, -0.03582763671875, 0.04150390625, + 0.003925323486328125, 0.00934600830078125, -0.022003173828125, + -0.004741668701171875, -0.01403045654296875, + -0.04925537109375, -0.0005216598510742188, + -0.0065765380859375, 0.01715087890625, -0.0152130126953125, + 0.0200042724609375, -0.0130615234375, -0.0202484130859375, + -0.0025310516357421875, 0.0059661865234375, + 0.0185699462890625, -0.0101776123046875, -0.021728515625, + -0.01114654541015625, -0.0189361572265625, 0.0223541259765625, + 0.0265960693359375, -0.02960205078125, 0.019287109375, + -0.01416778564453125, 0.0121307373046875, 0.034515380859375, + -0.033782958984375, 0.000728607177734375, -0.03411865234375, + 0.00010061264038085938, 0.0010194778442382812, + 0.0228271484375, 0.009246826171875, -0.0114288330078125, + -0.00408935546875, 0.00986480712890625, -0.016845703125, + -0.039886474609375, -0.00739288330078125, + 0.007213592529296875, 0.0256805419921875, 0.01123809814453125, + 0.009185791015625, -0.03607177734375, 0.0251617431640625, + -0.017852783203125, -0.007503509521484375, -0.037750244140625, + -0.02001953125, -0.01474761962890625, -0.0113372802734375, + 0.0115814208984375, -0.00036406517028808594, + -0.014190673828125, -0.0032482147216796875, + 0.0017681121826171875, -0.0238494873046875, + -0.025299072265625, -0.0113525390625, 0.01398468017578125, + 0.01377105712890625, 0.01200103759765625, 0.00659942626953125, + -0.0007700920104980469, -0.05926513671875, 0.0075531005859375, + -0.033935546875, 0.00881195068359375, 0.002941131591796875, + -0.02313232421875, 0.02215576171875, 0.06939697265625, + -0.0294647216796875, -0.0032405853271484375, + -0.00983428955078125, -0.023223876953125, + -0.00310516357421875, -0.0014495849609375, + 0.01334381103515625, -0.0032253265380859375, 0.033447265625, + -0.0108642578125, 0.020355224609375, 0.0001189112663269043, + -0.0002608299255371094, -0.0404052734375, + 0.0038318634033203125, 0.0113983154296875, + -0.0255584716796875, -0.00572967529296875, 0.0184783935546875, + -0.0272064208984375, 0.01013946533203125, + -0.01021575927734375, -0.0146636962890625, + -0.0161285400390625, -0.022796630859375, 0.026214599609375, + -0.036590576171875, -0.0227813720703125, 0.0120086669921875, + 0.03369140625, -0.04632568359375, -0.012939453125, + -0.03302001953125, -0.035858154296875, 0.0019168853759765625, + -0.006450653076171875, 0.0162506103515625, 0.0193023681640625, + -0.00664520263671875, -0.01361083984375, 0.04168701171875, + -0.047821044921875, 0.011138916015625, -0.0188751220703125, + 0.00011795759201049805, -0.0157470703125, -0.01470947265625, + -0.01255035400390625, 0.0099334716796875, -0.0283355712890625, + 0.053131103515625, -0.02044677734375, -0.027008056640625, + -0.05059814453125, -0.01421356201171875, + 0.0009927749633789062, 0.0079803466796875, 0.0181884765625, + 0.0401611328125, 0.01983642578125, -0.028778076171875, + -0.004673004150390625, 0.0011386871337890625, + 0.0002682209014892578, -0.00746917724609375, + -0.038360595703125, -0.01195526123046875, -0.037689208984375, + 0.039764404296875, -0.0227813720703125, -0.00922393798828125, + -0.038116455078125, 0.0167388916015625, 0.01528167724609375, + -0.0020923614501953125, -0.0123291015625, + -0.007373809814453125, -0.04132080078125, 0.03277587890625, + -0.0265045166015625, 0.033172607421875, 0.0306549072265625, + -0.0022735595703125, 0.0080718994140625, -0.0269317626953125, + 0.0202178955078125, 0.0038299560546875, 0.022674560546875, + 0.00959014892578125, 0.039520263671875, 0.00336456298828125, + -0.0233917236328125, 0.0082855224609375, 0.01971435546875, + -0.035430908203125, 0.05047607421875, 0.006145477294921875, + -0.05767822265625, 0.00943756103515625, -0.005275726318359375, + 0.0142822265625, -0.012359619140625, -0.007305145263671875, + 0.006893157958984375, -0.023101806640625, 0.050323486328125, + -0.0291595458984375, 0.0227203369140625, -0.035614013671875, + 0.0396728515625, 0.00412750244140625, 0.034393310546875, + 0.00354766845703125, -0.032135009765625, -0.0251617431640625, + 0.003032684326171875, -0.0310211181640625, -0.033660888671875, + 0.0011720657348632812, -0.009918212890625, + -0.0187530517578125, -0.038116455078125, -0.0288543701171875, + 0.05987548828125, 0.005603790283203125, -0.032379150390625, + 0.0026187896728515625, 0.01201629638671875, + 0.001361846923828125, 0.0191497802734375, -0.00958251953125, + 0.01006317138671875, -0.01715087890625, -0.0546875, + -0.00728607177734375, 0.0243072509765625, 0.043853759765625, + -0.004398345947265625, -0.0305938720703125, + -0.0135650634765625, -0.0357666015625, -0.01088714599609375, + 0.01580810546875, 0.0311126708984375, 0.01107025146484375, + 0.0009198188781738281, 0.01346588134765625, + -0.00598907470703125, -0.031585693359375, 0.0192718505859375, + -0.0298919677734375, 0.024322509765625, -0.0038299560546875, + -0.03668212890625, -0.03363037109375, 0.01568603515625, + 0.027618408203125, -0.0020351409912109375, 0.029083251953125, + 0.0151824951171875, -0.025604248046875, -0.01169586181640625, + -0.003437042236328125, -0.030426025390625, -0.004547119140625, + -0.027496337890625, -0.017486572265625, 0.016510009765625, + 0.049224853515625, -0.044769287109375, 0.0099945068359375, + 0.03118896484375, -0.033172607421875, 0.018585205078125, + -0.032806396484375, 0.02734375, -0.03875732421875, + -0.05328369140625, 0.0248870849609375, 0.0003790855407714844, + -0.0165863037109375, -0.0008473396301269531, + 0.006267547607421875, -0.0216522216796875, + -0.004703521728515625, 0.0313720703125, -0.004276275634765625, + -0.0088653564453125, -0.01087188720703125, 0.032623291015625, + -0.046142578125, -0.0183563232421875, -0.00930023193359375, + 0.040618896484375, 0.006988525390625, 0.024169921875, + 0.007266998291015625, 0.013397216796875, 0.00238800048828125, + 0.00870513916015625, 0.02020263671875, -0.033050537109375, + 0.054840087890625, 0.023162841796875, -0.0002655982971191406, + -0.050750732421875, -0.023345947265625, -0.01496124267578125, + 0.02587890625, 0.00466156005859375, -0.01174163818359375, + 0.0265960693359375, -0.0006241798400878906, + -0.0022716522216796875, -0.0010480880737304688, + -0.01158905029296875, 0.0088043212890625, 0.01372528076171875, + -0.0122833251953125, 0.0022296905517578125, + -0.0183258056640625, 0.0221405029296875, 0.0283355712890625, + 0.042083740234375, -0.00858306884765625, -0.034881591796875, + 0.0240020751953125, 0.037689208984375, -0.01555633544921875, + -0.039154052734375, 0.05511474609375, 0.01129150390625, + 0.0042266845703125, 0.0019359588623046875, + -0.0081329345703125, 0.01029205322265625, + -0.002567291259765625, -0.0022258758544921875, + -0.0098876953125, -0.013153076171875, -0.035858154296875, + 0.01329803466796875, -0.01081085205078125, 0.0192718505859375, + 0.00901031494140625, 0.00010645389556884766, + 0.0134429931640625, -0.0038509368896484375, 0.01763916015625, + 0.0283355712890625, 0.01259613037109375, 0.00826263427734375, + 0.04632568359375, -0.054901123046875, -0.051239013671875, + 0.030975341796875, 0.0223846435546875, 0.0031890869140625, + -0.01381683349609375, -0.0230712890625, -0.00873565673828125, + 0.036590576171875, -0.010467529296875, -0.03173828125, + -0.004344940185546875, -0.03387451171875, + -0.00930023193359375, 0.0092620849609375, -0.0279541015625, + 0.0113372802734375, 0.0222930908203125, 0.0179901123046875, + -0.006481170654296875, -0.01105499267578125, + -0.005947113037109375, 0.005802154541015625, -0.0125732421875, + -0.00921630859375, -0.0013303756713867188, + -0.0197906494140625, -0.005924224853515625, + -0.0263214111328125, -0.01259613037109375, + -0.005962371826171875, 0.017303466796875, -0.0138397216796875, + -0.042022705078125, 0.0295562744140625, -0.015716552734375, + 0.0291748046875, 0.035552978515625, 0.00788116455078125, + -0.01380157470703125, -0.0011653900146484375, + -0.0090179443359375, -0.001216888427734375, + 0.0025177001953125, 0.0003676414489746094, 0.0197906494140625, + -0.02069091796875, 0.01342010498046875, 0.0030536651611328125, + 0.03173828125, -0.001850128173828125, -0.052215576171875, + -0.023193359375, -0.012176513671875, -0.0166778564453125, + -0.03997802734375, -0.041229248046875, -0.017974853515625, + -0.00385284423828125, 0.005035400390625, + -0.002956390380859375, 0.033843994140625, -0.01116943359375, + 0.0279083251953125, -0.0178070068359375, 0.024627685546875, + 0.032867431640625, 0.003444671630859375, 0.0240478515625, + -0.01445770263671875, 0.040771484375, -0.01230621337890625, + 0.00666046142578125, -0.008087158203125, 0.006805419921875, + -0.004276275634765625, 0.0221405029296875, -0.024749755859375, + 0.0400390625, 0.0138397216796875, 0.0361328125, + -0.0028705596923828125, -0.003787994384765625, + 0.06475830078125, 0.0302886962890625, -0.043304443359375, + -0.04559326171875, 0.020751953125, -0.0275115966796875, + -0.02569580078125, -0.01329803466796875, 0.003986358642578125, + 0.04998779296875, -0.0101776123046875, -0.0172119140625, + 0.048004150390625, 0.014007568359375, -0.0207061767578125, + -0.0168914794921875, 0.0146026611328125, 0.0286712646484375, + 0.03729248046875, -0.0244598388671875, 0.00667572021484375, + -0.0207366943359375, 0.03302001953125, 0.00540924072265625, + -0.0284576416015625, 0.01849365234375, -0.011444091796875, + 0.001476287841796875, -0.005615234375, 0.03778076171875, + 0.02642822265625, 0.01239776611328125, -0.01181793212890625, + 0.03521728515625, 0.013031005859375, 0.01139068603515625, + 0.00672149658203125, 0.024444580078125, 0.0197601318359375, + -0.0177001953125, -0.024200439453125, 0.0216827392578125, + -0.0004317760467529297, 0.00811767578125, -0.0131683349609375, + 0.0012493133544921875, -0.01227569580078125, + 0.0030918121337890625, 0.041107177734375, 0.012481689453125, + -0.029998779296875, -0.0092620849609375, -0.01904296875, + 0.0090484619140625, -0.0301055908203125, -0.00742340087890625, + -0.01364898681640625, 0.0297088623046875, 0.0328369140625, + -0.0116729736328125, 0.00152587890625, 0.045867919921875, + 0.0035400390625, -0.03497314453125, 0.023406982421875, + 0.03289794921875, -0.028076171875, 0.01800537109375, + 0.01024627685546875, -0.004169464111328125, -0.00396728515625, + -0.0245513916015625, -0.01021575927734375, -0.031585693359375, + -0.05474853515625, 0.0168304443359375, 0.026519775390625, + 0.0267333984375, -0.01230621337890625, -0.0149383544921875, + 0.0204620361328125, 0.015380859375, -0.0626220703125, + -0.0162353515625, 0.0220794677734375, -0.0171356201171875, + -0.01561737060546875, -0.035980224609375, 0.0192718505859375, 0.043701171875, 0.0413818359375, 0.03314208984375, - 0.0001302957534790039, -0.00060272216796875, - -0.0177154541015625, -0.037384033203125, 0.037750244140625, - 0.0265045166015625, 0.01338958740234375, 0.0231170654296875, - 0.04119873046875, -0.0037441253662109375, -0.0226898193359375, - -0.000499725341796875, 0.00890350341796875, - -0.002899169921875, 0.007648468017578125, - 0.007366180419921875, -0.0113983154296875, - 0.003940582275390625, -0.007781982421875, -0.0400390625, - -0.0231781005859375, 0.007110595703125, -0.0287017822265625, - 0.032440185546875, 0.0176849365234375, -0.01515960693359375, - 0.00826263427734375, -0.0146636962890625, 0.01073455810546875, - 0.007213592529296875, 0.0014743804931640625, 0.03265380859375, - -0.00846099853515625, 0.03009033203125, 0.0013189315795898438, - -0.01103973388671875, -0.01983642578125, 0.0136566162109375, - -0.0312347412109375, -0.0228729248046875, 0.0274658203125, - 0.00033664703369140625, 0.050811767578125, 0.0234222412109375, - 0.011016845703125, -0.01800537109375, -0.0154876708984375, - 0.016876220703125, 0.006988525390625, 0.00920867919921875, - -0.01349639892578125, -0.0190887451171875, 0.028564453125, - 0.0092926025390625, 0.03021240234375, -0.01274871826171875, - 0.0049285888671875, 0.0024471282958984375, -0.06781005859375, - -0.007171630859375, 0.002124786376953125, 0.00334930419921875, - -0.0007500648498535156, 0.01947021484375, 0.0095367431640625, - -0.0262298583984375, -0.00958251953125, 0.00647735595703125, - 0.051727294921875, -0.0177154541015625, 0.0238189697265625, - -0.021453857421875, 0.030426025390625, -0.01262664794921875, - -0.0307159423828125, -0.037078857421875, 0.0223541259765625, - 0.03448486328125, -0.020599365234375, 0.0041046142578125, - -0.038787841796875, -0.0221710205078125, -0.0249786376953125, - -0.0214385986328125, -0.0003192424774169922, - 0.0036640167236328125, 0.0107574462890625, 0.0152435302734375, - -0.0162811279296875, -0.0222930908203125, 0.01052093505859375, - -0.0185699462890625, -0.01507568359375, 0.013458251953125, - 0.0023326873779296875, 0.00759124755859375, - -0.0022640228271484375, 0.0276336669921875, - 0.0083465576171875, 0.0155181884765625, -0.00701904296875, - 0.00833892822265625, -0.01355743408203125, 0.02069091796875, - 0.050933837890625, 0.01476287841796875, 0.0205841064453125, - -0.004924774169921875, 0.038299560546875, 0.0301361083984375, - 0.055267333984375, 0.00765228271484375, 0.00782012939453125, - -0.0016546249389648438, 0.0191497802734375, - 0.0026912689208984375, -0.0172119140625, 0.0219879150390625, - -0.0187835693359375, 0.035064697265625, -0.010101318359375, - -0.048065185546875, 0.0150604248046875, -0.031707763671875, - 0.0182647705078125, 0.02392578125, 0.043548583984375, - 0.01558685302734375, -0.0275115966796875, 0.01690673828125, - -0.003688812255859375, 0.01544952392578125, - -0.0037899017333984375, -0.0030841827392578125, - 0.0343017578125, -0.003437042236328125, -0.0285797119140625, - -0.03607177734375, -0.0008149147033691406, -0.01275634765625, - -0.036376953125, -0.007778167724609375, -0.04376220703125, - 0.027099609375, -0.05804443359375, -0.0144195556640625, - 0.0072479248046875, 0.045013427734375, 0.007221221923828125, - 0.025390625, 0.054718017578125, 0.00618743896484375, - -0.0198974609375, 0.0101470947265625, -0.035919189453125, - 0.00145721435546875, -0.00609588623046875, -0.027557373046875, - -0.01454925537109375, -0.0137176513671875, -0.002197265625, - -0.0038280487060546875, 0.006763458251953125, - 0.00432586669921875, -0.0203094482421875, 0.0181732177734375, - -0.00463104248046875, -0.0200042724609375, 0.03448486328125, - -0.0709228515625, -0.028533935546875, 0.0260162353515625, - -0.044677734375, 0.0212249755859375, 0.032501220703125, - 0.0163116455078125, -0.01091766357421875, + 0.00016868114471435547, -0.000579833984375, + -0.0177154541015625, -0.037384033203125, 0.0377197265625, + 0.0264739990234375, 0.01340484619140625, 0.02313232421875, + 0.04119873046875, -0.0037288665771484375, -0.0226287841796875, + -0.0005016326904296875, 0.00885772705078125, + -0.0029125213623046875, 0.007633209228515625, 0.00732421875, + -0.0113983154296875, 0.00392913818359375, + -0.00774383544921875, -0.040008544921875, -0.0231781005859375, + 0.007091522216796875, -0.0286865234375, 0.032440185546875, + 0.0177001953125, -0.015167236328125, 0.0082550048828125, + -0.01465606689453125, 0.01073455810546875, + 0.007236480712890625, 0.00146484375, 0.03265380859375, + -0.0084686279296875, 0.0301361083984375, 0.0013427734375, + -0.01102447509765625, -0.0197906494140625, + 0.01364898681640625, -0.031219482421875, -0.0229034423828125, + 0.0275115966796875, 0.0003364086151123047, 0.050811767578125, + 0.0234222412109375, 0.01097869873046875, -0.017974853515625, + -0.0154876708984375, 0.016876220703125, 0.0070037841796875, + 0.00920867919921875, -0.0135040283203125, -0.019073486328125, + 0.028564453125, 0.00928497314453125, 0.030181884765625, + -0.01273345947265625, 0.00492095947265625, + 0.0024242401123046875, -0.06781005859375, -0.00714111328125, + 0.0021343231201171875, 0.003330230712890625, + -0.0007662773132324219, 0.019500732421875, 0.0095062255859375, + -0.0262298583984375, -0.00960540771484375, + 0.006473541259765625, 0.05169677734375, -0.017730712890625, + 0.0238037109375, -0.0214691162109375, 0.0304412841796875, + -0.0126190185546875, -0.03070068359375, -0.037078857421875, + 0.022369384765625, 0.034515380859375, -0.0205841064453125, + 0.004116058349609375, -0.038787841796875, -0.0221710205078125, + -0.024993896484375, -0.021453857421875, + -0.0003190040588378906, 0.0036602020263671875, + 0.0107269287109375, 0.0152435302734375, -0.016265869140625, + -0.0222930908203125, 0.0105133056640625, -0.0185546875, + -0.015045166015625, 0.0134735107421875, 0.002307891845703125, + 0.007579803466796875, -0.0022735595703125, 0.0276336669921875, + 0.0083160400390625, 0.01552581787109375, + -0.007038116455078125, 0.008331298828125, -0.0135955810546875, + 0.020721435546875, 0.050933837890625, 0.0147552490234375, + 0.0205841064453125, -0.00490570068359375, 0.038299560546875, + 0.0301055908203125, 0.05523681640625, 0.00766754150390625, + 0.00782012939453125, -0.00164794921875, 0.0191497802734375, + 0.002674102783203125, -0.017242431640625, 0.02203369140625, + -0.0187835693359375, 0.0350341796875, -0.01007080078125, + -0.04803466796875, 0.01506805419921875, -0.031646728515625, + 0.0182342529296875, 0.0239105224609375, 0.0435791015625, + 0.0156097412109375, -0.0275115966796875, 0.0169525146484375, + -0.0036716461181640625, 0.0154571533203125, + -0.00377655029296875, -0.003070831298828125, + 0.034332275390625, -0.0034275054931640625, + -0.0285797119140625, -0.03607177734375, + -0.0008091926574707031, -0.0127410888671875, -0.036376953125, + -0.007793426513671875, -0.043792724609375, 0.0270538330078125, + -0.058013916015625, -0.01438140869140625, + 0.007251739501953125, 0.045013427734375, 0.00720977783203125, + 0.0254058837890625, 0.054718017578125, 0.0061798095703125, + -0.0198822021484375, 0.010162353515625, -0.035919189453125, + 0.0014743804931640625, -0.0060577392578125, -0.02752685546875, + -0.01453399658203125, -0.0137481689453125, + -0.00218963623046875, -0.0038661956787109375, + 0.006755828857421875, 0.00434112548828125, -0.020294189453125, + 0.0181884765625, -0.004611968994140625, -0.0200042724609375, + 0.034454345703125, -0.0709228515625, -0.028533935546875, + 0.0260772705078125, -0.044677734375, 0.021240234375, + 0.032470703125, 0.0162811279296875, -0.01093292236328125, -0.01284027099609375, -0.028900146484375, - 0.0009822845458984375, -0.0187835693359375, - 0.0098114013671875, 0.0172119140625, 0.0307159423828125, - 0.004657745361328125, -0.0228271484375, 0.01180267333984375, - -0.028472900390625, -0.0211639404296875, -0.03936767578125, - 0.0176239013671875, -0.038238525390625, -0.03961181640625, - 0.0026645660400390625, -0.01483154296875, - 0.005336761474609375, 0.00011247396469116211, - 0.01042938232421875, -0.01023101806640625, -0.020721435546875, - -0.0256500244140625, 0.022186279296875, -0.01110076904296875, - 0.0234222412109375, -0.0209197998046875, - -0.0036983489990234375, -0.037628173828125, - 0.0189056396484375, 0.0112762451171875, -0.0162200927734375, - 0.00176239013671875, 0.01241302490234375, - -0.0038700103759765625, -0.0017251968383789062, - 0.043731689453125, -0.0298004150390625, 0.0050811767578125, - -0.0214385986328125, -0.0266265869140625, - -0.01386260986328125, 0.006877899169921875, - -0.0007877349853515625, 0.03021240234375, 0.051544189453125, - -0.04302978515625, -0.037811279296875, -0.0004911422729492188, - 0.0103607177734375, 0.0136260986328125, 0.025604248046875, - 0.0211639404296875, -0.0160675048828125, -0.022796630859375, - -0.004734039306640625, 0.013763427734375, - -0.01389312744140625, 0.0256195068359375, + 0.0009975433349609375, -0.0187835693359375, + 0.0098419189453125, 0.0171966552734375, 0.030670166015625, + 0.00463104248046875, -0.022796630859375, 0.0117950439453125, + -0.028472900390625, -0.021148681640625, -0.039337158203125, + 0.0176544189453125, -0.038238525390625, -0.039581298828125, + 0.002685546875, -0.01483154296875, 0.00533294677734375, + 0.0000928044319152832, 0.01045989990234375, -0.01025390625, + -0.0207061767578125, -0.025665283203125, 0.0222015380859375, + -0.0111083984375, 0.023406982421875, -0.0209197998046875, + -0.0036716461181640625, -0.037689208984375, + 0.0189056396484375, 0.0112457275390625, -0.016204833984375, + 0.0017604827880859375, 0.01241302490234375, + -0.0038776397705078125, -0.00168609619140625, 0.043701171875, + -0.0297698974609375, 0.005054473876953125, + -0.0214385986328125, -0.026611328125, -0.01385498046875, + 0.00682830810546875, -0.0007939338684082031, + 0.030181884765625, 0.051513671875, -0.04296875, + -0.037811279296875, -0.0005049705505371094, + 0.01035308837890625, 0.0136260986328125, 0.025604248046875, + 0.0211334228515625, -0.01605224609375, -0.0228118896484375, + -0.004764556884765625, 0.01377105712890625, + -0.01389312744140625, 0.0256805419921875, 0.0035343170166015625, -0.026397705078125, - -0.0199432373046875, -0.018646240234375, -0.0250701904296875, - 0.0275421142578125, 0.0146026611328125, -0.002964019775390625, - 0.03839111328125, -0.0166168212890625, 0.00832366943359375, - 0.060455322265625, 0.0234527587890625, 0.0117950439453125, - 0.006397247314453125, -0.014068603515625, - -0.007488250732421875, -0.00616455078125, -0.0210723876953125, - 0.0007615089416503906, -0.01064300537109375, 0.00775146484375, - 0.016632080078125, 0.01084136962890625, -0.01355743408203125, - 0.0091552734375, 0.03436279296875, -0.013916015625, - -0.037353515625, -0.029632568359375, 0.018524169921875, - -0.0230712890625, 0.01107025146484375, -0.021484375, - 0.0247955322265625, -0.00873565673828125, 0.0184783935546875, - 0.00930023193359375, 0.0202484130859375, 0.0223541259765625, - 0.00608062744140625, 0.035430908203125, -0.035614013671875, - -0.0004277229309082031, -0.003662109375, 0.00539398193359375, - 0.0024127960205078125, 0.007221221923828125, - -0.0256805419921875, 0.00926971435546875, 0.00698089599609375, - -0.0077667236328125, 0.014862060546875, -0.031402587890625, - 0.0160675048828125, -0.0022525787353515625, - 0.01468658447265625, 0.01129913330078125, 0.019622802734375, - -0.0193939208984375, 0.00928497314453125, 0.013824462890625, - 0.0013227462768554688, 0.0131072998046875, - -0.0032062530517578125, -0.0093841552734375, - 0.0257415771484375, 0.0307464599609375, 0.02386474609375, - 0.0018949508666992188, -0.01251220703125, -0.010223388671875, - -0.035614013671875, 0.0228118896484375, - -0.000028192996978759766, 0.0123138427734375, 0.005126953125, - -0.010589599609375, 0.00533294677734375, 0.009490966796875, - 0.00566864013671875, -0.005893707275390625, - 0.01282501220703125, -0.0113372802734375, 0.00579071044921875, - 0.01239776611328125, -0.0068359375, 0.04034423828125, - -0.011077880859375, -0.0012264251708984375, 0.015899658203125, - -0.0450439453125, -0.01468658447265625, 0.0083160400390625, - -0.0121612548828125, -0.0027618408203125, - 0.0014629364013671875, -0.004535675048828125, - -0.0034885406494140625, -0.01180267333984375, 0.0029296875, - -0.025909423828125, 0.00821685791015625, - -0.0020427703857421875, 0.014404296875, 0.036346435546875, - -0.01197052001953125, -0.03778076171875, 0.0245819091796875, - -0.035858154296875, 0.00846099853515625, 0.0182342529296875, - 0.0088653564453125, -0.0467529296875, 0.01409912109375, - 0.01203155517578125, -0.0232696533203125, -0.0079498291015625, - 0.0123291015625, -0.0310516357421875, 0.00372314453125, - 0.0136260986328125, 0.00312042236328125, - 0.0007691383361816406, 0.004894256591796875 + -0.0199737548828125, -0.018646240234375, -0.025054931640625, + 0.02752685546875, 0.01459503173828125, -0.002986907958984375, + 0.038421630859375, -0.0166168212890625, 0.00832366943359375, + 0.06048583984375, 0.0234527587890625, 0.01180267333984375, + 0.00638580322265625, -0.0140838623046875, -0.0074615478515625, + -0.00616455078125, -0.02105712890625, 0.000720977783203125, + -0.01064300537109375, 0.00774383544921875, 0.016632080078125, + 0.0108795166015625, -0.0135498046875, 0.00914764404296875, + 0.034332275390625, -0.01390838623046875, -0.037353515625, + -0.0296173095703125, 0.01849365234375, -0.0230255126953125, + 0.01108551025390625, -0.0214691162109375, 0.0247955322265625, + -0.00872802734375, 0.0184478759765625, 0.00928497314453125, + 0.0202484130859375, 0.0223541259765625, 0.006092071533203125, + 0.035430908203125, -0.035614013671875, -0.0004284381866455078, + -0.0036907196044921875, 0.0053863525390625, + 0.002460479736328125, 0.00722503662109375, + -0.0257110595703125, 0.00926971435546875, 0.00696563720703125, + -0.007740020751953125, 0.01490020751953125, -0.03143310546875, + 0.0160980224609375, -0.0022335052490234375, 0.0146484375, + 0.01129913330078125, 0.019683837890625, -0.0193634033203125, + 0.0092620849609375, 0.01381683349609375, 0.001338958740234375, + 0.01309967041015625, -0.0032100677490234375, + -0.00939178466796875, 0.02569580078125, 0.030731201171875, + 0.023895263671875, 0.0019083023071289062, -0.0124969482421875, + -0.01023101806640625, -0.03558349609375, 0.022857666015625, + -0.000028312206268310547, 0.0123291015625, 0.005157470703125, + -0.01056671142578125, 0.005367279052734375, 0.009490966796875, + 0.005687713623046875, -0.00585174560546875, + 0.01279449462890625, -0.01137542724609375, + 0.005809783935546875, 0.0124359130859375, + -0.006900787353515625, 0.04034423828125, -0.0111083984375, + -0.0012426376342773438, 0.0159454345703125, -0.0450439453125, + -0.01468658447265625, 0.00832366943359375, + -0.01213836669921875, -0.0028018951416015625, + 0.0014476776123046875, -0.00453948974609375, + -0.003498077392578125, -0.0117645263671875, + 0.00293731689453125, -0.0258941650390625, 0.008209228515625, + -0.00203704833984375, 0.0144195556640625, 0.036346435546875, + -0.0119476318359375, -0.03778076171875, 0.0245819091796875, + -0.035858154296875, 0.0084991455078125, 0.01824951171875, + 0.0088653564453125, -0.0467529296875, 0.01412200927734375, + 0.01203155517578125, -0.0232391357421875, + -0.00788116455078125, 0.0123443603515625, -0.031036376953125, + 0.003704071044921875, 0.0136260986328125, + 0.0031147003173828125, 0.0007843971252441406, + 0.00490570068359375 ], "index": 0, "object": "embedding" @@ -969,13 +966,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d93f0a165b0d-VIE", + "cf-ray": "9fdb13334cd4d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:05 GMT", + "date": "Mon, 18 May 2026 13:05:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "169", + "openai-processing-ms": "74", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -990,7 +987,7 @@ "x-ratelimit-remaining-tokens": "9999993", "x-ratelimit-reset-requests": "6ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_a5706d98d85f494384df565a8e8603b6" + "x-request-id": "req_28ae5ebacb5444d88174500d4efbe1d9" }, "status": 200, "statusText": "OK" @@ -1000,7 +997,7 @@ "callIndex": 3, "id": "83cbbf6da482c905", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:06.392Z", + "recordedAt": "2026-05-18T13:05:43.930Z", "request": { "body": { "kind": "json", @@ -1058,11 +1055,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845026, - "created_at": 1778845025, + "completed_at": 1779109543, + "created_at": 1779109543, "error": null, "frequency_penalty": 0, - "id": "resp_0dc1f4a60163e85d006a0705617e748197a42250f97f6c5e11", + "id": "resp_03e8e419765c8277006a0b0ea72f208195ba6063651ead5e40", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1074,8 +1071,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_8Bm4Dn0ZNvq2D7SzUtC9vDDa", - "id": "fc_0dc1f4a60163e85d006a070561ff4c81979a3edfe928475646", + "call_id": "call_cVjyNB3MSiQRTMUkXTMi1nVB", + "id": "fc_03e8e419765c8277006a0b0ea79b208195904fbcadc5d1555d", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1142,13 +1139,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d940acc35b0d-VIE", + "cf-ray": "9fdb13343f81d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:06 GMT", + "date": "Mon, 18 May 2026 13:05:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "883", + "openai-processing-ms": "752", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1159,10 +1156,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999687", + "x-ratelimit-remaining-tokens": "149999690", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_31f9b246eca94996ad391599a6625618" + "x-request-id": "req_1515347d78c54b548c31c815a9896a6a" }, "status": 200, "statusText": "OK" @@ -1172,7 +1169,7 @@ "callIndex": 4, "id": "417e64addcf59da7", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:08.727Z", + "recordedAt": "2026-05-18T13:05:44.946Z", "request": { "body": { "kind": "json", @@ -1192,11 +1189,11 @@ "role": "user" }, { - "id": "fc_0dc1f4a60163e85d006a070561ff4c81979a3edfe928475646", + "id": "fc_03e8e419765c8277006a0b0ea79b208195904fbcadc5d1555d", "type": "item_reference" }, { - "call_id": "call_8Bm4Dn0ZNvq2D7SzUtC9vDDa", + "call_id": "call_cVjyNB3MSiQRTMUkXTMi1nVB", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1239,11 +1236,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845028, - "created_at": 1778845026, + "completed_at": 1779109544, + "created_at": 1779109544, "error": null, "frequency_penalty": 0, - "id": "resp_0dc1f4a60163e85d006a070562a1348197b88cc56272b15992", + "id": "resp_03e8e419765c8277006a0b0ea81d508195be0980a16597faf4", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1255,8 +1252,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_q1Jwt5cAdTEpVkFX3oDaXh5B", - "id": "fc_0dc1f4a60163e85d006a0705645e6081978741ca754b0d8301", + "call_id": "call_wkD9OHeGZwAYLZmYbZqfr1de", + "id": "fc_03e8e419765c8277006a0b0ea897708195b0653bf0b76fc67e", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1323,13 +1320,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d947ae585b0d-VIE", + "cf-ray": "9fdb133a0e35d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:08 GMT", + "date": "Mon, 18 May 2026 13:05:45 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "2095", + "openai-processing-ms": "835", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1343,7 +1340,7 @@ "x-ratelimit-remaining-tokens": "149999650", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_13a96346c7ca406daf502b2f649cee7d" + "x-request-id": "req_3a4d65bf579d4f4aa2cd82a92e261c7c" }, "status": 200, "statusText": "OK" @@ -1353,7 +1350,7 @@ "callIndex": 5, "id": "341675a0a1a88c63", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:10.594Z", + "recordedAt": "2026-05-18T13:05:46.114Z", "request": { "body": { "kind": "json", @@ -1373,20 +1370,20 @@ "role": "user" }, { - "id": "fc_0dc1f4a60163e85d006a070561ff4c81979a3edfe928475646", + "id": "fc_03e8e419765c8277006a0b0ea79b208195904fbcadc5d1555d", "type": "item_reference" }, { - "call_id": "call_8Bm4Dn0ZNvq2D7SzUtC9vDDa", + "call_id": "call_cVjyNB3MSiQRTMUkXTMi1nVB", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0dc1f4a60163e85d006a0705645e6081978741ca754b0d8301", + "id": "fc_03e8e419765c8277006a0b0ea897708195b0653bf0b76fc67e", "type": "item_reference" }, { - "call_id": "call_q1Jwt5cAdTEpVkFX3oDaXh5B", + "call_id": "call_wkD9OHeGZwAYLZmYbZqfr1de", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1429,11 +1426,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845030, - "created_at": 1778845029, + "completed_at": 1779109545, + "created_at": 1779109545, "error": null, "frequency_penalty": 0, - "id": "resp_0dc1f4a60163e85d006a07056533548197922cba98cf592d58", + "id": "resp_03e8e419765c8277006a0b0ea91e50819586d4d76fa14fd374", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1445,8 +1442,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_yI2hHnvvVuqSSskGGqap2CWT", - "id": "fc_0dc1f4a60163e85d006a07056628d88197ac7ee69f4f0f4877", + "call_id": "call_e3Ivj2FAVKuHa0YhmElaO6xE", + "id": "fc_03e8e419765c8277006a0b0ea9b2d081959f2d84629990e5c6", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1513,13 +1510,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d9561ec15b0d-VIE", + "cf-ray": "9fdb13405e72d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:10 GMT", + "date": "Mon, 18 May 2026 13:05:46 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "1549", + "openai-processing-ms": "996", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1530,10 +1527,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999612", + "x-ratelimit-remaining-tokens": "149999610", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_2750d9cb7c6e93fa9eeeb9cfbec67e8e" + "x-request-id": "req_acc559a248bc4956a04f7aba5888c3d5" }, "status": 200, "statusText": "OK" @@ -1543,7 +1540,7 @@ "callIndex": 6, "id": "cbb9070d68f84d1b", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:14.509Z", + "recordedAt": "2026-05-18T13:05:47.947Z", "request": { "body": { "kind": "json", @@ -1563,29 +1560,29 @@ "role": "user" }, { - "id": "fc_0dc1f4a60163e85d006a070561ff4c81979a3edfe928475646", + "id": "fc_03e8e419765c8277006a0b0ea79b208195904fbcadc5d1555d", "type": "item_reference" }, { - "call_id": "call_8Bm4Dn0ZNvq2D7SzUtC9vDDa", + "call_id": "call_cVjyNB3MSiQRTMUkXTMi1nVB", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0dc1f4a60163e85d006a0705645e6081978741ca754b0d8301", + "id": "fc_03e8e419765c8277006a0b0ea897708195b0653bf0b76fc67e", "type": "item_reference" }, { - "call_id": "call_q1Jwt5cAdTEpVkFX3oDaXh5B", + "call_id": "call_wkD9OHeGZwAYLZmYbZqfr1de", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0dc1f4a60163e85d006a07056628d88197ac7ee69f4f0f4877", + "id": "fc_03e8e419765c8277006a0b0ea9b2d081959f2d84629990e5c6", "type": "item_reference" }, { - "call_id": "call_yI2hHnvvVuqSSskGGqap2CWT", + "call_id": "call_e3Ivj2FAVKuHa0YhmElaO6xE", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1628,11 +1625,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845034, - "created_at": 1778845030, + "completed_at": 1779109547, + "created_at": 1779109546, "error": null, "frequency_penalty": 0, - "id": "resp_0dc1f4a60163e85d006a070566cb588197afafb4f3e0e09c54", + "id": "resp_03e8e419765c8277006a0b0eaa8aa48195bc0b7ee30c3ee717", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1644,8 +1641,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_E6kWpvSYzU56djSz1IYVGCcb", - "id": "fc_0dc1f4a60163e85d006a07056a215481978ae7b0e9582f04c2", + "call_id": "call_06QylJav4QrEdhZmKBPJLK7i", + "id": "fc_03e8e419765c8277006a0b0eab518c81959f3dc74159ad35b9", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1712,13 +1709,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d961cd7b5b0d-VIE", + "cf-ray": "9fdb1347b8fad87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:14 GMT", + "date": "Mon, 18 May 2026 13:05:47 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "3702", + "openai-processing-ms": "1586", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1732,7 +1729,7 @@ "x-ratelimit-remaining-tokens": "149999572", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_60fb9e8044344caebae76f134cb7c834" + "x-request-id": "req_4466f5eb2cc349c886d7188c98b38776" }, "status": 200, "statusText": "OK" @@ -1742,7 +1739,7 @@ "callIndex": 7, "id": "bccc54d212e2503d", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:16.335Z", + "recordedAt": "2026-05-18T13:05:49.149Z", "request": { "body": { "kind": "json", @@ -1776,11 +1773,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845036, - "created_at": 1778845034, + "completed_at": 1779109548, + "created_at": 1779109548, "error": null, "frequency_penalty": 0, - "id": "resp_0b5b21700e73bb60006a07056abfd881908329703150f73a70", + "id": "resp_02875295f868a141006a0b0eac29308195abc78ae85eee07e8", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -1799,7 +1796,7 @@ "type": "output_text" } ], - "id": "msg_0b5b21700e73bb60006a07056c121c81908aea136c83b17bd7", + "id": "msg_02875295f868a141006a0b0eacd7e8819582108852804f7955", "role": "assistant", "status": "completed", "type": "message" @@ -1848,13 +1845,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d97a58de5b0d-VIE", + "cf-ray": "9fdb13535a18d87b-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:16 GMT", + "date": "Mon, 18 May 2026 13:05:49 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "1592", + "openai-processing-ms": "991", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1868,7 +1865,7 @@ "x-ratelimit-remaining-tokens": "149996362", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_56d3918163a54f09bb1e7e809d9ad82d" + "x-request-id": "req_317b06e5652e4c188240ada79caeb1e8" }, "status": 200, "statusText": "OK" @@ -1878,7 +1875,7 @@ "callIndex": 8, "id": "8eec740c1be01c5c", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:23.301Z", + "recordedAt": "2026-05-18T13:05:55.137Z", "request": { "body": { "kind": "json", @@ -1912,11 +1909,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845043, - "created_at": 1778845041, + "completed_at": 1779109554, + "created_at": 1779109554, "error": null, "frequency_penalty": 0, - "id": "resp_02fc4e6b40326e06006a070571c4c881938c483fa05b8890d8", + "id": "resp_01a0dad14c18bba6006a0b0eb26f5c8190a9e5b2db1a184af0", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -1935,7 +1932,7 @@ "type": "output_text" } ], - "id": "msg_02fc4e6b40326e06006a07057312208193907df5e12d9bc9d2", + "id": "msg_01a0dad14c18bba6006a0b0eb2d44c81908a65cd04324b2ed7", "role": "assistant", "status": "completed", "type": "message" @@ -1984,13 +1981,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d9a61e2e2182-VIE", + "cf-ray": "9fdb137a7e8834c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:23 GMT", + "date": "Mon, 18 May 2026 13:05:55 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "1514", + "openai-processing-ms": "706", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2004,7 +2001,7 @@ "x-ratelimit-remaining-tokens": "149996365", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_a1ac6a3eb0274e10bd66a26bca78d9dc" + "x-request-id": "req_dda8295207224ba782d2e3d0723f9552" }, "status": 200, "statusText": "OK" @@ -2014,7 +2011,7 @@ "callIndex": 9, "id": "6945bf18d345ad4a", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:29.285Z", + "recordedAt": "2026-05-18T13:06:02.937Z", "request": { "body": { "kind": "json", @@ -2048,11 +2045,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845048, - "created_at": 1778845048, + "completed_at": 1779109562, + "created_at": 1779109560, "error": null, "frequency_penalty": 0, - "id": "resp_0c4124ddc8f1b684006a0705789ddc8195922d6a2bb23b4777", + "id": "resp_01f587ae9e154dfa006a0b0eb8747c8193b8d67e4cf6dc1c18", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2071,7 +2068,7 @@ "type": "output_text" } ], - "id": "msg_0c4124ddc8f1b684006a070578f70c8195b4c1bb6925c9ff06", + "id": "msg_01f587ae9e154dfa006a0b0ebab1088193885c1b95395ee9c2", "role": "assistant", "status": "completed", "type": "message" @@ -2120,13 +2117,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d9d10ac15b91-VIE", + "cf-ray": "9fdb139fd8ee5b20-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:29 GMT", + "date": "Mon, 18 May 2026 13:06:02 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "544", + "openai-processing-ms": "2463", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2140,7 +2137,7 @@ "x-ratelimit-remaining-tokens": "149996365", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_74ae2250743948b092cbabd4f99cc676" + "x-request-id": "req_3871ab194e2147249d076ae663b4701d" }, "status": 200, "statusText": "OK" @@ -2150,7 +2147,7 @@ "callIndex": 0, "id": "fcd9d51c145fb90b", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:37:30.030Z", + "recordedAt": "2026-05-18T13:06:04.031Z", "request": { "body": { "kind": "json", @@ -2188,7 +2185,7 @@ "type": "text" } ], - "id": "msg_01KguK9P4wR6BTYfS8f1PYjM", + "id": "msg_01E5UdgVw7ebRBeQKpzjZXZD", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2213,31 +2210,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:37:29Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:06:03Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:37:29Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:06:03Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:37:29Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:06:03Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:37:29Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:06:03Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d9d70eac6c45-VIE", + "cf-ray": "9fdb13b178ebc2bb-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:30 GMT", - "request-id": "req_011Cb4Hdkm9CYUssaKMPHSZj", + "date": "Mon, 18 May 2026 13:06:04 GMT", + "request-id": "req_011CbA5pCi1Dgdv5PkNBHmXa", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-122e64a894f6f037b34f57836bd5f8ac-a23c61b53aa4340a-01", + "traceresponse": "00-2e415c86490bf0135c9ae7c8b6cfc082-c9e3ed7cd37005ef-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "506", + "x-envoy-upstream-service-time": "805", "x-robots-tag": "none" }, "status": 200, @@ -2248,7 +2245,7 @@ "callIndex": 1, "id": "ff22a182ea5a9c06", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:37:35.793Z", + "recordedAt": "2026-05-18T13:06:09.951Z", "request": { "body": { "kind": "json", @@ -2286,7 +2283,7 @@ "type": "text" } ], - "id": "msg_01LDT7f7TTR7EojhDdCH25kt", + "id": "msg_01NVtGZiyhtk5ejEhnqA62NL", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2311,31 +2308,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:37:35Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:06:09Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:37:35Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:06:09Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:37:35Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:06:09Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:37:35Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:06:09Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1d9fb19dec2ff-VIE", + "cf-ray": "9fdb13d768452942-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:35 GMT", - "request-id": "req_011Cb4HeBSwwmVqttqxExtjm", + "date": "Mon, 18 May 2026 13:06:10 GMT", + "request-id": "req_011CbA5peKdEmKAfYbKupqgy", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-51ed35e5ba665e13c3041d5ceea2cb88-3b48e1d43cd87d25-01", + "traceresponse": "00-d70acee53678976765dea1a29840283c-23e791ac8a9baa41-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "478", + "x-envoy-upstream-service-time": "640", "x-robots-tag": "none" }, "status": 200, @@ -2346,7 +2343,7 @@ "callIndex": 2, "id": "f1d50a1e6c760d6c", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:37:43.332Z", + "recordedAt": "2026-05-18T13:06:16.022Z", "request": { "body": { "kind": "json", @@ -2384,7 +2381,7 @@ "type": "text" } ], - "id": "msg_01Etwq5SsyCCLTwobfdgE2Wx", + "id": "msg_019gGJuX7dbarRXqM3rDvXj7", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2409,31 +2406,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:37:43Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:06:15Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:37:43Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:06:15Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:37:41Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:06:15Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:37:43Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:06:15Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da1f29ecc5fa-VIE", + "cf-ray": "9fdb13fc6b8d84ce-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:43 GMT", - "request-id": "req_011Cb4HeciELd1tXwQoXsc38", + "date": "Mon, 18 May 2026 13:06:16 GMT", + "request-id": "req_011CbA5q5betypq5mmke4RMN", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-edd92473fe0f3f01a599a0ed36441e0f-b1e900c0c4aa3c0f-01", + "traceresponse": "00-7be2ef35282fa4d2645183654eae7de0-a98336518e983da4-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2258", + "x-envoy-upstream-service-time": "805", "x-robots-tag": "none" }, "status": 200, @@ -2444,7 +2441,7 @@ "callIndex": 10, "id": "ab34eb26595033a1", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:44.224Z", + "recordedAt": "2026-05-18T13:06:16.946Z", "request": { "body": { "kind": "json", @@ -2477,11 +2474,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845064, - "created_at": 1778845063, + "completed_at": 1779109576, + "created_at": 1779109576, "error": null, "frequency_penalty": 0, - "id": "resp_08029bee53bc08b2006a0705879bf88193b4671b00d16a4291", + "id": "resp_0715e78b9c31e316006a0b0ec83e1881968b9bdf71a55dd9a9", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2500,7 +2497,7 @@ "type": "output_text" } ], - "id": "msg_08029bee53bc08b2006a07058800708193bf586f2358ded534", + "id": "msg_0715e78b9c31e316006a0b0ec8b1dc8196ba03fec64360aae2", "role": "assistant", "status": "completed", "type": "message" @@ -2549,13 +2546,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da2ec9245aa7-VIE", + "cf-ray": "9fdb1402eb3034c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:44 GMT", + "date": "Mon, 18 May 2026 13:06:17 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "591", + "openai-processing-ms": "695", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2566,10 +2563,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999962", + "x-ratelimit-remaining-tokens": "149999965", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_e4faef13c75b4197a2a290c53793c763" + "x-request-id": "req_48d0453870f144d18718f81cbac7a07a" }, "status": 200, "statusText": "OK" @@ -2579,7 +2576,7 @@ "callIndex": 11, "id": "250e64f864c72337", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:45.231Z", + "recordedAt": "2026-05-18T13:06:17.897Z", "request": { "body": { "kind": "json", @@ -2630,11 +2627,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845065, - "created_at": 1778845064, + "completed_at": 1779109577, + "created_at": 1779109577, "error": null, "frequency_penalty": 0, - "id": "resp_01f7ed5a80e8837c006a0705886ea88197bdaa9b53a99bad7a", + "id": "resp_01c20678d6b82f69006a0b0ec9232481949e15dad870996cd1", "incomplete_details": null, "instructions": null, "max_output_tokens": 32, @@ -2653,7 +2650,7 @@ "type": "output_text" } ], - "id": "msg_01f7ed5a80e8837c006a070588f050819798cefc00c9b8e209", + "id": "msg_01c20678d6b82f69006a0b0ec9a848819494298d03a6b2a4e1", "role": "assistant", "status": "completed", "type": "message" @@ -2715,13 +2712,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da33fefb5aa7-VIE", + "cf-ray": "9fdb14086e0434c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:45 GMT", + "date": "Mon, 18 May 2026 13:06:17 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "789", + "openai-processing-ms": "761", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2732,10 +2729,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999942", + "x-ratelimit-remaining-tokens": "149999945", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_19c1bd52f6d74f0ebf2def4b5d7a4cc5" + "x-request-id": "req_f14bb76d288346febf0fa9555acd3720" }, "status": 200, "statusText": "OK" @@ -2745,7 +2742,7 @@ "callIndex": 12, "id": "fecb7c9f8d04e1bd", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:46.175Z", + "recordedAt": "2026-05-18T13:06:19.064Z", "request": { "body": { "kind": "json", @@ -2792,19 +2789,19 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_01573d08966d2d84006a0705896e308190a87831e156786420\",\"object\":\"response\",\"created_at\":1778845065,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_01573d08966d2d84006a0705896e308190a87831e156786420\",\"object\":\"response\",\"created_at\":1778845065,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"{\\\"\",\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"obfuscation\":\"2NsskLOACPBRNk\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"city\",\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"obfuscation\":\"mAEi6Cv7ZchD\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\":\\\"\",\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"obfuscation\":\"54a8mNNfJSVS3\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"obfuscation\":\"cDlRJPKvaU6\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"}\",\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"obfuscation\":\"UlyHofs1DsYaz5\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":9,\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"},\"sequence_number\":10}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":11}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_01573d08966d2d84006a0705896e308190a87831e156786420\",\"object\":\"response\",\"created_at\":1778845065,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845065,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_01573d08966d2d84006a070589dca88190a5c5b0e455252bca\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":38,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":44},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_09d5abf859d0da91006a0b0eca175c81969e0a4fe26e2b0cac\",\"object\":\"response\",\"created_at\":1779109578,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_09d5abf859d0da91006a0b0eca175c81969e0a4fe26e2b0cac\",\"object\":\"response\",\"created_at\":1779109578,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"{\\\"\",\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"obfuscation\":\"0R8qyLC9BaCKRD\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"city\",\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"obfuscation\":\"vWttCc5Dy6AR\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\":\\\"\",\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"obfuscation\":\"h6sKuuy2G20ch\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"obfuscation\":\"ARUyGXSXFr1\",\"output_index\":0,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"}\",\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"obfuscation\":\"LdMKmIbQToblLe\",\"output_index\":0,\"sequence_number\":8}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":9,\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"},\"sequence_number\":10}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":11}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_09d5abf859d0da91006a0b0eca175c81969e0a4fe26e2b0cac\",\"object\":\"response\",\"created_at\":1779109578,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109578,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_09d5abf859d0da91006a0b0ecaa26081969196aa8bf0d23a6b\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":38,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":44},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" ], "kind": "sse" }, @@ -2812,12 +2809,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da3a4ef35aa7-VIE", + "cf-ray": "9fdb140e5a4534c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:37:45 GMT", + "date": "Mon, 18 May 2026 13:06:18 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "91", + "openai-processing-ms": "213", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2825,7 +2822,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_e20d7fb0faec40e3809761cc1ec1ff79" + "x-request-id": "req_40161b0de6914a9aa008e70e261a2b73" }, "status": 200, "statusText": "OK" @@ -2835,7 +2832,7 @@ "callIndex": 13, "id": "eebc152ba05b8412", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:51.017Z", + "recordedAt": "2026-05-18T13:06:20.296Z", "request": { "body": { "kind": "json", @@ -2871,11 +2868,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845070, - "created_at": 1778845066, + "completed_at": 1779109580, + "created_at": 1779109579, "error": null, "frequency_penalty": 0, - "id": "resp_0d2c71760a47d0c2006a07058a64388193b7a28a374af481f4", + "id": "resp_0afe7a32d6f7e112006a0b0ecb40d88196a0189db7d0834838", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2894,7 +2891,7 @@ "type": "output_text" } ], - "id": "msg_0d2c71760a47d0c2006a07058ea8688193a7332c731caee945", + "id": "msg_0afe7a32d6f7e112006a0b0ecc017081969ab6c0fb05668266", "role": "assistant", "status": "completed", "type": "message" @@ -2943,13 +2940,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da402e4d5aa7-VIE", + "cf-ray": "9fdb1415ab2d34c2-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:37:51 GMT", + "date": "Mon, 18 May 2026 13:06:20 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "4608", + "openai-processing-ms": "1042", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2963,7 +2960,7 @@ "x-ratelimit-remaining-tokens": "149999955", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_58116cf1376e465386fcb234c09ec4bd" + "x-request-id": "req_176e5a4816ad42a686012bad175bb698" }, "status": 200, "statusText": "OK" @@ -2973,7 +2970,7 @@ "callIndex": 14, "id": "bc08e707f2920073", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:37:55.851Z", + "recordedAt": "2026-05-18T13:06:21.198Z", "request": { "body": { "kind": "json", @@ -3005,17 +3002,17 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_01c69d092f6a4ad2006a07058f358c8193b1dbdb4baab095d8\",\"object\":\"response\",\"created_at\":1778845071,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_01c69d092f6a4ad2006a07058f358c8193b1dbdb4baab095d8\",\"object\":\"response\",\"created_at\":1778845071,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"STREAM\",\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"logprobs\":[],\"obfuscation\":\"5x7PYlN1Lj\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" HEL\",\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"logprobs\":[],\"obfuscation\":\"oXNWlbBTm5zs\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"LO\",\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"logprobs\":[],\"obfuscation\":\"bAxEhTe6bWKRQy\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"STREAM HELLO\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_01c69d092f6a4ad2006a07058f358c8193b1dbdb4baab095d8\",\"object\":\"response\",\"created_at\":1778845071,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845075,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_01c69d092f6a4ad2006a070593924881938d93bf4cfb587a4e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":27,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":31},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_01eb9c65f08d36cd006a0b0ecc78908193a067f38e1aeb55eb\",\"object\":\"response\",\"created_at\":1779109580,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_01eb9c65f08d36cd006a0b0ecc78908193a067f38e1aeb55eb\",\"object\":\"response\",\"created_at\":1779109580,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"STREAM\",\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"logprobs\":[],\"obfuscation\":\"EM7JybqIXr\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" HEL\",\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"logprobs\":[],\"obfuscation\":\"MILw2JEJ1RSH\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"LO\",\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"logprobs\":[],\"obfuscation\":\"NhxEWOQ6KdrhcT\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"STREAM HELLO\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"},\"sequence_number\":8}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_01eb9c65f08d36cd006a0b0ecc78908193a067f38e1aeb55eb\",\"object\":\"response\",\"created_at\":1779109580,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109580,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_01eb9c65f08d36cd006a0b0ecce3d88193a750ca54bb1f3808\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":27,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":31},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" ], "kind": "sse" }, @@ -3023,12 +3020,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1da5e6f975aa7-VIE", + "cf-ray": "9fdb141d5f8134c2-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:37:51 GMT", + "date": "Mon, 18 May 2026 13:06:20 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "115", + "openai-processing-ms": "88", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -3036,7 +3033,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_5b7c96e4d0b541df992e5c3ce43756b1" + "x-request-id": "req_1fcc04829378403990f52c6efbb49f78" }, "status": 200, "statusText": "OK" diff --git a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v6.cassette.json b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v6.cassette.json index c7fde0cc8..0507c9124 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v6.cassette.json +++ b/e2e/scenarios/ai-sdk-instrumentation/__cassettes__/ai-sdk-v6.cassette.json @@ -4,7 +4,7 @@ "callIndex": 0, "id": "7c0e162d6794cf5d", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:45.976Z", + "recordedAt": "2026-05-18T13:07:07.304Z", "request": { "body": { "kind": "json", @@ -37,11 +37,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845125, - "created_at": 1778845125, + "completed_at": 1779109626, + "created_at": 1779109626, "error": null, "frequency_penalty": 0, - "id": "resp_0d2b3dded95f545e006a0705c541848190b55923d8743f5988", + "id": "resp_0b7bfff11d07a42b006a0b0efa133c8195a8b150f93b18b318", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -60,7 +60,7 @@ "type": "output_text" } ], - "id": "msg_0d2b3dded95f545e006a0705c5a9248190b89170479a9f94ba", + "id": "msg_0b7bfff11d07a42b006a0b0efabfb88195bf852fee8c787e06", "role": "assistant", "status": "completed", "type": "message" @@ -109,13 +109,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbb039a15b7f-VIE", + "cf-ray": "9fdb153a3ce85a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:46 GMT", + "date": "Mon, 18 May 2026 13:07:07 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "714", + "openai-processing-ms": "1228", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -126,10 +126,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999965", + "x-ratelimit-remaining-tokens": "149999962", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_069f93a8841f4366acbe661b287a73f2" + "x-request-id": "req_fe9ddedd882044688b25ab77ac7c9b4d" }, "status": 200, "statusText": "OK" @@ -139,7 +139,7 @@ "callIndex": 1, "id": "f12aff3913439248", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:46.833Z", + "recordedAt": "2026-05-18T13:07:08.402Z", "request": { "body": { "kind": "json", @@ -190,11 +190,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845126, - "created_at": 1778845126, + "completed_at": 1779109628, + "created_at": 1779109627, "error": null, "frequency_penalty": 0, - "id": "resp_0789f52b62d7de3c006a0705c62bbc8193a8873b7c45998026", + "id": "resp_07c830358a333e1d006a0b0efb7e40819089bde9f478007bff", "incomplete_details": null, "instructions": null, "max_output_tokens": 32, @@ -213,7 +213,7 @@ "type": "output_text" } ], - "id": "msg_0789f52b62d7de3c006a0705c699248193bea3e1811565eeda", + "id": "msg_07c830358a333e1d006a0b0efc0dec8190be19fc3706b281d2", "role": "assistant", "status": "completed", "type": "message" @@ -275,13 +275,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbb5fee05b7f-VIE", + "cf-ray": "9fdb15433be55a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:46 GMT", + "date": "Mon, 18 May 2026 13:07:08 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "662", + "openai-processing-ms": "905", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -295,7 +295,7 @@ "x-ratelimit-remaining-tokens": "149999935", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_52159e2375194fa0b5d8bd15fe6c1694" + "x-request-id": "req_8d3b3d65214746fcbdfda0e11648bcb7" }, "status": 200, "statusText": "OK" @@ -305,7 +305,7 @@ "callIndex": 2, "id": "5d2312f425df9f2f", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:49.350Z", + "recordedAt": "2026-05-18T13:07:09.496Z", "request": { "body": { "kind": "json", @@ -334,20 +334,20 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0060f6a1544b3f20006a0705c7095c81979020be31674f199b\",\"object\":\"response\",\"created_at\":1778845127,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0060f6a1544b3f20006a0705c7095c81979020be31674f199b\",\"object\":\"response\",\"created_at\":1778845127,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"One\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"P5gZjcgGO413q\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"ANNClaCv9J7VKjB\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" two\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"HF65n1XrATQD\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"PqJEElPafjVY8Ki\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"G1QlAT0LYH\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"obfuscation\":\"iUi8iOUgIl6GRQK\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":10,\"text\":\"One, two, three.\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"},\"sequence_number\":11}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":12}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0060f6a1544b3f20006a0705c7095c81979020be31674f199b\",\"object\":\"response\",\"created_at\":1778845127,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845127,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0060f6a1544b3f20006a0705c77c6c819797677454a0cab7aa\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":29},\"user\":null,\"metadata\":{}},\"sequence_number\":13}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0046b53281f5e7cf006a0b0efc9720819396a98b98caea65b3\",\"object\":\"response\",\"created_at\":1779109628,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0046b53281f5e7cf006a0b0efc9720819396a98b98caea65b3\",\"object\":\"response\",\"created_at\":1779109628,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"One\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"vLGNjpL2apkUB\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"DWD1aHDdcI0uEbn\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" two\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"m98UXEMV8CWF\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"PPsHywALwfje50M\",\"output_index\":0,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"LtsJrnV8Dx\",\"output_index\":0,\"sequence_number\":8}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"obfuscation\":\"08aQhjxpBOx8sDQ\",\"output_index\":0,\"sequence_number\":9}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":10,\"text\":\"One, two, three.\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"},\"sequence_number\":11}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":12}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0046b53281f5e7cf006a0b0efc9720819396a98b98caea65b3\",\"object\":\"response\",\"created_at\":1779109628,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109629,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0046b53281f5e7cf006a0b0efd30d08193a3bb1dedf7090d1a\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"One, two, three.\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":29},\"user\":null,\"metadata\":{}},\"sequence_number\":13}" ], "kind": "sse" }, @@ -355,12 +355,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbbb4caf5b7f-VIE", + "cf-ray": "9fdb154a08345a53-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:38:47 GMT", + "date": "Mon, 18 May 2026 13:07:08 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "225", + "openai-processing-ms": "179", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -368,7 +368,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_74e351abd37d478399510488d3021e08" + "x-request-id": "req_989ec12d404d462a84c577d62329bd7a" }, "status": 200, "statusText": "OK" @@ -378,7 +378,7 @@ "callIndex": 0, "id": "7d3c0bb6426e6be4", "matchKey": "POST api.openai.com/v1/embeddings", - "recordedAt": "2026-05-15T11:38:49.504Z", + "recordedAt": "2026-05-18T13:07:09.770Z", "request": { "body": { "kind": "json", @@ -966,13 +966,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbcb1d385b7f-VIE", + "cf-ray": "9fdb1550ed925a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:49 GMT", + "date": "Mon, 18 May 2026 13:07:09 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "59", + "openai-processing-ms": "186", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -987,7 +987,7 @@ "x-ratelimit-remaining-tokens": "9999993", "x-ratelimit-reset-requests": "6ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_d4f4ed4488844e83aad6be51d138995a" + "x-request-id": "req_41e260f5ea2d467cb90c43d49719b712" }, "status": 200, "statusText": "OK" @@ -997,7 +997,7 @@ "callIndex": 0, "id": "98922b41de07d721", "matchKey": "POST api.cohere.com/v2/rerank", - "recordedAt": "2026-05-15T11:38:49.793Z", + "recordedAt": "2026-05-18T13:07:10.080Z", "request": { "body": { "kind": "json", @@ -1020,7 +1020,7 @@ "body": { "kind": "json", "value": { - "id": "92496624-9bfc-4ac0-bb26-a7027d13b7d7", + "id": "1a312981-2ef0-42f8-bbb6-956a281fd120", "meta": { "api_version": { "version": "2" @@ -1047,7 +1047,7 @@ "cache-control": "no-cache, no-store, no-transform, must-revalidate, private, max-age=0", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:49 GMT", + "date": "Mon, 18 May 2026 13:07:10 GMT", "expires": "Thu, 01 Jan 1970 00:00:00 GMT", "pragma": "no-cache", "server": "envoy", @@ -1055,8 +1055,8 @@ "vary": "Origin,Accept-Encoding", "via": "1.1 google", "x-accel-expires": "0", - "x-debug-trace-id": "df6198b518a963e1a5a8eb92139f169e", - "x-envoy-upstream-service-time": "53" + "x-debug-trace-id": "fb69e72d3d05e7d78b401f247fa407e4", + "x-envoy-upstream-service-time": "54" }, "status": 200, "statusText": "OK" @@ -1066,7 +1066,7 @@ "callIndex": 3, "id": "83cbbf6da482c905", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:50.972Z", + "recordedAt": "2026-05-18T13:07:11.616Z", "request": { "body": { "kind": "json", @@ -1123,11 +1123,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845130, - "created_at": 1778845130, + "completed_at": 1779109631, + "created_at": 1779109630, "error": null, "frequency_penalty": 0, - "id": "resp_0404a219ec47b246006a0705ca01208196ad5824a698e7eb39", + "id": "resp_019b35a2e5b500be006a0b0efe45388195a325a20ec0381074", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1139,8 +1139,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_FdCTcBCUxIxTWMhFhYDYGKA3", - "id": "fc_0404a219ec47b246006a0705ca8f648196a8dd8edd390cc8ee", + "call_id": "call_sXYAVqsiN19asQObBL0kyj6F", + "id": "fc_019b35a2e5b500be006a0b0eff387c8195b65e79940db9ffbe", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1207,13 +1207,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbcdc8235b7f-VIE", + "cf-ray": "9fdb1554886b5a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:51 GMT", + "date": "Mon, 18 May 2026 13:07:11 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "963", + "openai-processing-ms": "1342", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1227,7 +1227,7 @@ "x-ratelimit-remaining-tokens": "149999687", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_d6b7300691e84e6d9cf256a366e6dbb8" + "x-request-id": "req_5621afd716e741918f8d0604cf369bac" }, "status": 200, "statusText": "OK" @@ -1237,7 +1237,7 @@ "callIndex": 4, "id": "417e64addcf59da7", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:52.157Z", + "recordedAt": "2026-05-18T13:07:13.100Z", "request": { "body": { "kind": "json", @@ -1257,11 +1257,11 @@ "role": "user" }, { - "id": "fc_0404a219ec47b246006a0705ca8f648196a8dd8edd390cc8ee", + "id": "fc_019b35a2e5b500be006a0b0eff387c8195b65e79940db9ffbe", "type": "item_reference" }, { - "call_id": "call_FdCTcBCUxIxTWMhFhYDYGKA3", + "call_id": "call_sXYAVqsiN19asQObBL0kyj6F", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1303,11 +1303,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845131, - "created_at": 1778845131, + "completed_at": 1779109632, + "created_at": 1779109631, "error": null, "frequency_penalty": 0, - "id": "resp_0404a219ec47b246006a0705cb30148196b380469777b7abd8", + "id": "resp_019b35a2e5b500be006a0b0effced88195ab653960f656ab91", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1319,8 +1319,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_ow94ONjPsMpZ0rMB2Ck9dkTK", - "id": "fc_0404a219ec47b246006a0705cbc6488196a891eaccf8b05976", + "call_id": "call_XglVqVVLI8Rkc1I28RiZyLca", + "id": "fc_019b35a2e5b500be006a0b0f00b87881959b35df50ec6a7625", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1387,13 +1387,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbd528ce5b7f-VIE", + "cf-ray": "9fdb155e1fa55a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:52 GMT", + "date": "Mon, 18 May 2026 13:07:13 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "953", + "openai-processing-ms": "1291", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1407,7 +1407,7 @@ "x-ratelimit-remaining-tokens": "149999650", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_68a6cfa7aae542d189f21074aa7f6049" + "x-request-id": "req_790a4ff156054e818b9e1d3c8a186a33" }, "status": 200, "statusText": "OK" @@ -1417,7 +1417,7 @@ "callIndex": 5, "id": "341675a0a1a88c63", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:53.263Z", + "recordedAt": "2026-05-18T13:07:14.386Z", "request": { "body": { "kind": "json", @@ -1437,20 +1437,20 @@ "role": "user" }, { - "id": "fc_0404a219ec47b246006a0705ca8f648196a8dd8edd390cc8ee", + "id": "fc_019b35a2e5b500be006a0b0eff387c8195b65e79940db9ffbe", "type": "item_reference" }, { - "call_id": "call_FdCTcBCUxIxTWMhFhYDYGKA3", + "call_id": "call_sXYAVqsiN19asQObBL0kyj6F", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0404a219ec47b246006a0705cbc6488196a891eaccf8b05976", + "id": "fc_019b35a2e5b500be006a0b0f00b87881959b35df50ec6a7625", "type": "item_reference" }, { - "call_id": "call_ow94ONjPsMpZ0rMB2Ck9dkTK", + "call_id": "call_XglVqVVLI8Rkc1I28RiZyLca", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1492,11 +1492,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845133, - "created_at": 1778845132, + "completed_at": 1779109634, + "created_at": 1779109633, "error": null, "frequency_penalty": 0, - "id": "resp_0404a219ec47b246006a0705cc5c408196bb7377f340e86045", + "id": "resp_019b35a2e5b500be006a0b0f014d1c8195a2c27f2a50bfae99", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1508,8 +1508,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_gCQdgCxiFSafXdWd8V8N1Ceg", - "id": "fc_0404a219ec47b246006a0705ccf8a0819689389802c82591f1", + "call_id": "call_9C5Hr12zyxc416vblymWOYyK", + "id": "fc_019b35a2e5b500be006a0b0f01f3888195a64a5ec83aa4be6d", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1576,13 +1576,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbdc98795b7f-VIE", + "cf-ray": "9fdb15676e485a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:53 GMT", + "date": "Mon, 18 May 2026 13:07:14 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "894", + "openai-processing-ms": "1085", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1593,10 +1593,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149999610", + "x-ratelimit-remaining-tokens": "149999612", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_d31b42b74def49f1b10b8437ff43f145" + "x-request-id": "req_659df726ec304d3db0ce5777f7cba898" }, "status": 200, "statusText": "OK" @@ -1606,7 +1606,7 @@ "callIndex": 6, "id": "cbb9070d68f84d1b", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:54.705Z", + "recordedAt": "2026-05-18T13:07:15.391Z", "request": { "body": { "kind": "json", @@ -1626,29 +1626,29 @@ "role": "user" }, { - "id": "fc_0404a219ec47b246006a0705ca8f648196a8dd8edd390cc8ee", + "id": "fc_019b35a2e5b500be006a0b0eff387c8195b65e79940db9ffbe", "type": "item_reference" }, { - "call_id": "call_FdCTcBCUxIxTWMhFhYDYGKA3", + "call_id": "call_sXYAVqsiN19asQObBL0kyj6F", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0404a219ec47b246006a0705cbc6488196a891eaccf8b05976", + "id": "fc_019b35a2e5b500be006a0b0f00b87881959b35df50ec6a7625", "type": "item_reference" }, { - "call_id": "call_ow94ONjPsMpZ0rMB2Ck9dkTK", + "call_id": "call_XglVqVVLI8Rkc1I28RiZyLca", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" }, { - "id": "fc_0404a219ec47b246006a0705ccf8a0819689389802c82591f1", + "id": "fc_019b35a2e5b500be006a0b0f01f3888195a64a5ec83aa4be6d", "type": "item_reference" }, { - "call_id": "call_gCQdgCxiFSafXdWd8V8N1Ceg", + "call_id": "call_9C5Hr12zyxc416vblymWOYyK", "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", "type": "function_call_output" } @@ -1690,11 +1690,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845134, - "created_at": 1778845133, + "completed_at": 1779109635, + "created_at": 1779109634, "error": null, "frequency_penalty": 0, - "id": "resp_0404a219ec47b246006a0705cd76b48196961b30173130d354", + "id": "resp_019b35a2e5b500be006a0b0f0293fc8195a0546fff1761e23e", "incomplete_details": null, "instructions": null, "max_output_tokens": 128, @@ -1706,8 +1706,8 @@ "output": [ { "arguments": "{\"location\":\"Paris, France\"}", - "call_id": "call_qGcDkXqvbKBOLP1gkHgwWtBk", - "id": "fc_0404a219ec47b246006a0705ce6dbc819692fa89a3c85f82b0", + "call_id": "call_zdHIoq2m6uggecWEeOfB4ElL", + "id": "fc_019b35a2e5b500be006a0b0f030030819588e1859ceca1fb59", "name": "get_weather", "status": "completed", "type": "function_call" @@ -1774,13 +1774,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbe378225b7f-VIE", + "cf-ray": "9fdb156f7c2e5a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:54 GMT", + "date": "Mon, 18 May 2026 13:07:15 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "1243", + "openai-processing-ms": "812", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1794,7 +1794,7 @@ "x-ratelimit-remaining-tokens": "149999572", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_8e1c85e31fb344d2bd593ffb692a1008" + "x-request-id": "req_32babc0c68ad4bd3ae9cdfc4f2beb018" }, "status": 200, "statusText": "OK" @@ -1804,7 +1804,7 @@ "callIndex": 7, "id": "bccc54d212e2503d", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:38:55.618Z", + "recordedAt": "2026-05-18T13:07:16.873Z", "request": { "body": { "kind": "json", @@ -1838,11 +1838,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845135, - "created_at": 1778845134, + "completed_at": 1779109636, + "created_at": 1779109636, "error": null, "frequency_penalty": 0, - "id": "resp_0e038950ec20b2fe006a0705ceef748194865d806c806ff0a4", + "id": "resp_0cc727c931a9b376006a0b0f0396808193b25f3b09ab696d8f", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -1861,7 +1861,7 @@ "type": "output_text" } ], - "id": "msg_0e038950ec20b2fe006a0705cf56308194b19b6a92abcee970", + "id": "msg_0cc727c931a9b376006a0b0f048eac8193921f0000a04522d8", "role": "assistant", "status": "completed", "type": "message" @@ -1910,13 +1910,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dbec99e85b7f-VIE", + "cf-ray": "9fdb1575c8365a53-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:38:55 GMT", + "date": "Mon, 18 May 2026 13:07:16 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "684", + "openai-processing-ms": "1277", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -1927,10 +1927,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149996362", + "x-ratelimit-remaining-tokens": "149996365", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_870d2894df384642b7383c4299aa4efb" + "x-request-id": "req_c4726fdc41374ce4a0c7648835e6e228" }, "status": 200, "statusText": "OK" @@ -1940,7 +1940,7 @@ "callIndex": 8, "id": "8eec740c1be01c5c", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:01.552Z", + "recordedAt": "2026-05-18T13:07:22.932Z", "request": { "body": { "kind": "json", @@ -1974,11 +1974,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845141, - "created_at": 1778845140, + "completed_at": 1779109642, + "created_at": 1779109642, "error": null, "frequency_penalty": 0, - "id": "resp_002fcd7088a6e368006a0705d4f30881908495300d227e79c4", + "id": "resp_06399320d0a09412006a0b0f0a257c81958a9edacf3828a6f6", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -1997,7 +1997,7 @@ "type": "output_text" } ], - "id": "msg_002fcd7088a6e368006a0705d552ec8190882719e4e79e4c3f", + "id": "msg_06399320d0a09412006a0b0f0a84f48195bf3af7ab4be8b2bc", "role": "assistant", "status": "completed", "type": "message" @@ -2046,13 +2046,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc122b9ac28c-VIE", + "cf-ray": "9fdb159eb8d325c0-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:01 GMT", + "date": "Mon, 18 May 2026 13:07:22 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "580", + "openai-processing-ms": "785", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2063,10 +2063,10 @@ "x-ratelimit-limit-requests": "30000", "x-ratelimit-limit-tokens": "150000000", "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "149996362", + "x-ratelimit-remaining-tokens": "149996365", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_41a30b5ba9c24fddbe8d2bda3a1f4a54" + "x-request-id": "req_d53e7b436df74ee6a2a8a0c26dec479b" }, "status": 200, "statusText": "OK" @@ -2076,7 +2076,7 @@ "callIndex": 9, "id": "6945bf18d345ad4a", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:07.512Z", + "recordedAt": "2026-05-18T13:07:29.329Z", "request": { "body": { "kind": "json", @@ -2110,11 +2110,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845147, - "created_at": 1778845146, + "completed_at": 1779109649, + "created_at": 1779109648, "error": null, "frequency_penalty": 0, - "id": "resp_07e868210009fc30006a0705dad50881968582b3f630ef55ba", + "id": "resp_0cbe689e44b6c604006a0b0f1038d8819694d8634b48810db7", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2133,7 +2133,7 @@ "type": "output_text" } ], - "id": "msg_07e868210009fc30006a0705db3d388196b0159232764ad89b", + "id": "msg_0cbe689e44b6c604006a0b0f10e93481968e88d27d49278b7c", "role": "assistant", "status": "completed", "type": "message" @@ -2182,13 +2182,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc370f9fc28c-VIE", + "cf-ray": "9fdb15c48d4170b0-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:07 GMT", + "date": "Mon, 18 May 2026 13:07:29 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "661", + "openai-processing-ms": "1120", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2202,7 +2202,7 @@ "x-ratelimit-remaining-tokens": "149996365", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_2173f28e7f8e4d999e5edef1519d58c0" + "x-request-id": "req_0750e0887301480faf51336a6f9393ec" }, "status": 200, "statusText": "OK" @@ -2212,7 +2212,7 @@ "callIndex": 0, "id": "fcd9d51c145fb90b", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:39:08.241Z", + "recordedAt": "2026-05-18T13:07:30.222Z", "request": { "body": { "kind": "json", @@ -2250,7 +2250,7 @@ "type": "text" } ], - "id": "msg_01SMnj8Kvm283yaiCWHJw6jH", + "id": "msg_01RwnKbK6MEukeBNtdLRZGyr", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2275,31 +2275,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:39:08Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:07:30Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:39:08Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:07:30Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:39:07Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:07:29Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:39:08Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:07:30Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc3cfa40a0c4-VIE", + "cf-ray": "9fdb15cd5a3b5b39-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:08 GMT", - "request-id": "req_011Cb4HkzkDLxccs2NWFZU4r", + "date": "Mon, 18 May 2026 13:07:30 GMT", + "request-id": "req_011CbA5vZmhE2Z7TAhvLsTgD", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-b6cce74f1d13c0e9f0607136864fcb51-7e3d88d4f1f362f7-01", + "traceresponse": "00-b72671940ca728e7ea27fe4660fdeb44-673e3c2e53738cf8-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "479", + "x-envoy-upstream-service-time": "623", "x-robots-tag": "none" }, "status": 200, @@ -2310,7 +2310,7 @@ "callIndex": 1, "id": "ff22a182ea5a9c06", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:39:14.071Z", + "recordedAt": "2026-05-18T13:07:36.182Z", "request": { "body": { "kind": "json", @@ -2348,7 +2348,7 @@ "type": "text" } ], - "id": "msg_017vizi3WKZF8nXGFtMYwubY", + "id": "msg_018RwWwgFKDqoLhZSR91RkHN", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2373,31 +2373,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:39:14Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:07:36Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:39:14Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:07:36Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:39:13Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:07:35Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:39:14Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:07:36Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc60fc325ba0-VIE", + "cf-ray": "9fdb15f21bf7dfcc-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:14 GMT", - "request-id": "req_011Cb4HmRR2NEQaYcNT6Ws58", + "date": "Mon, 18 May 2026 13:07:36 GMT", + "request-id": "req_011CbA5vzqKe1CK7mqLDXPWW", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-260a2e468990d334dff735a7282d3d8c-5c53933420ef05c1-01", + "traceresponse": "00-3e648882edd4a95f13f00ff193a19e69-607206b02a2eb091-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "539", + "x-envoy-upstream-service-time": "689", "x-robots-tag": "none" }, "status": 200, @@ -2408,7 +2408,7 @@ "callIndex": 2, "id": "f1d50a1e6c760d6c", "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-15T11:39:19.815Z", + "recordedAt": "2026-05-18T13:07:42.064Z", "request": { "body": { "kind": "json", @@ -2446,7 +2446,7 @@ "type": "text" } ], - "id": "msg_01XrUtKEEPa4zGUoFQ7L5tUc", + "id": "msg_01AE6GVgDnaMYmu5rh5WP14U", "model": "claude-haiku-4-5-20251001", "role": "assistant", "stop_details": null, @@ -2471,31 +2471,31 @@ "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", "anthropic-ratelimit-input-tokens-limit": "4000000", "anthropic-ratelimit-input-tokens-remaining": "4000000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-15T11:39:19Z", + "anthropic-ratelimit-input-tokens-reset": "2026-05-18T13:07:41Z", "anthropic-ratelimit-output-tokens-limit": "800000", "anthropic-ratelimit-output-tokens-remaining": "800000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-15T11:39:19Z", + "anthropic-ratelimit-output-tokens-reset": "2026-05-18T13:07:42Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-15T11:39:19Z", + "anthropic-ratelimit-requests-reset": "2026-05-18T13:07:41Z", "anthropic-ratelimit-tokens-limit": "4800000", "anthropic-ratelimit-tokens-remaining": "4800000", - "anthropic-ratelimit-tokens-reset": "2026-05-15T11:39:19Z", + "anthropic-ratelimit-tokens-reset": "2026-05-18T13:07:41Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc855a5cf832-VIE", + "cf-ray": "9fdb16176d9d5b83-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:19 GMT", - "request-id": "req_011Cb4HmrF1oG4g3fz9jTNju", + "date": "Mon, 18 May 2026 13:07:42 GMT", + "request-id": "req_011CbA5wSLFRBp1GBQ6HtqhW", "server": "cloudflare", "set-cookie": "[REDACTED]", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-5d914c986326051b338e958943a35cad-66b12d5c21a81587-01", + "traceresponse": "00-13a789fed95c61b64c33a12858885f25-f3c29c89b582b38b-01", "transfer-encoding": "chunked", "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "477", + "x-envoy-upstream-service-time": "619", "x-robots-tag": "none" }, "status": 200, @@ -2506,7 +2506,7 @@ "callIndex": 10, "id": "ab34eb26595033a1", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:20.924Z", + "recordedAt": "2026-05-18T13:07:43.080Z", "request": { "body": { "kind": "json", @@ -2539,11 +2539,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845160, - "created_at": 1778845160, + "completed_at": 1779109662, + "created_at": 1779109662, "error": null, "frequency_penalty": 0, - "id": "resp_00d991524dd65f70006a0705e81b1c81938f002fb667c418fc", + "id": "resp_039ea97737086498006a0b0f1e4d4881969bf84fc6b0041997", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2562,7 +2562,7 @@ "type": "output_text" } ], - "id": "msg_00d991524dd65f70006a0705e8abd48193b73f34e5a64d76c1", + "id": "msg_039ea97737086498006a0b0f1ec9308196ad54dc9b9058011b", "role": "assistant", "status": "completed", "type": "message" @@ -2611,13 +2611,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc89bf0fc2b6-VIE", + "cf-ray": "9fdb161cbdf6a0c4-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:20 GMT", + "date": "Mon, 18 May 2026 13:07:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "813", + "openai-processing-ms": "777", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2631,7 +2631,7 @@ "x-ratelimit-remaining-tokens": "149999965", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_fdc0ca990d2144e39cbb441b4ebf4963" + "x-request-id": "req_30004eafed6648cc816cf104adad8263" }, "status": 200, "statusText": "OK" @@ -2641,7 +2641,7 @@ "callIndex": 11, "id": "250e64f864c72337", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:21.876Z", + "recordedAt": "2026-05-18T13:07:43.884Z", "request": { "body": { "kind": "json", @@ -2692,11 +2692,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845161, - "created_at": 1778845161, + "completed_at": 1779109663, + "created_at": 1779109663, "error": null, "frequency_penalty": 0, - "id": "resp_0e0478cd0d3fb9bb006a0705e9253c8194b91b48a7091d76ff", + "id": "resp_03f93df3df146b93006a0b0f1f43f081939d0225ebb23e3910", "incomplete_details": null, "instructions": null, "max_output_tokens": 32, @@ -2715,7 +2715,7 @@ "type": "output_text" } ], - "id": "msg_0e0478cd0d3fb9bb006a0705e982008194a3552e97d74cc26a", + "id": "msg_03f93df3df146b93006a0b0f1f9574819392c49dfdcd22546e", "role": "assistant", "status": "completed", "type": "message" @@ -2777,13 +2777,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc906e66c2b6-VIE", + "cf-ray": "9fdb1622ca61a0c4-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:21 GMT", + "date": "Mon, 18 May 2026 13:07:43 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "721", + "openai-processing-ms": "627", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2797,7 +2797,7 @@ "x-ratelimit-remaining-tokens": "149999942", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_0751a216548e415eb41b6b7e88bbfd20" + "x-request-id": "req_eeca664fb00341eab42d20bf201ead79" }, "status": 200, "statusText": "OK" @@ -2807,7 +2807,7 @@ "callIndex": 12, "id": "fecb7c9f8d04e1bd", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:22.942Z", + "recordedAt": "2026-05-18T13:07:44.694Z", "request": { "body": { "kind": "json", @@ -2854,19 +2854,19 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0689d25ecf2acaf7006a0705ea16ec819787c2c23c04d952a8\",\"object\":\"response\",\"created_at\":1778845162,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0689d25ecf2acaf7006a0705ea16ec819787c2c23c04d952a8\",\"object\":\"response\",\"created_at\":1778845162,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"{\\\"\",\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"obfuscation\":\"T0g8p4ghZQIBc1\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"city\",\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"obfuscation\":\"G8TMkPPkSMjp\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\":\\\"\",\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"obfuscation\":\"jWppwhiXeCXGI\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"obfuscation\":\"UD0cCODV5I9\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"}\",\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"obfuscation\":\"IjJPwbP7Jybbui\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":9,\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"},\"sequence_number\":10}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":11}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0689d25ecf2acaf7006a0705ea16ec819787c2c23c04d952a8\",\"object\":\"response\",\"created_at\":1778845162,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845162,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0689d25ecf2acaf7006a0705ea96d4819789ce14c4ebd1df5e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":38,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":44},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0ef2b9c5ebc1c507006a0b0f2014548194adab9aa95a8a826f\",\"object\":\"response\",\"created_at\":1779109664,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0ef2b9c5ebc1c507006a0b0f2014548194adab9aa95a8a826f\",\"object\":\"response\",\"created_at\":1779109664,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"{\\\"\",\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"obfuscation\":\"jd2dvIGAkFeNJM\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"city\",\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"obfuscation\":\"FQTF3ZweHRyD\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\":\\\"\",\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"obfuscation\":\"VW98j4SpRMzyV\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"obfuscation\":\"PGbDfrYuDOz\",\"output_index\":0,\"sequence_number\":7}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"}\",\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"obfuscation\":\"9ZjCBTh5DC9YbV\",\"output_index\":0,\"sequence_number\":8}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":9,\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"},\"sequence_number\":10}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":11}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0ef2b9c5ebc1c507006a0b0f2014548194adab9aa95a8a826f\",\"object\":\"response\",\"created_at\":1779109664,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109664,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0ef2b9c5ebc1c507006a0b0f2072c4819482633bcde8618502\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"{\\\"city\\\":\\\"Paris\\\"}\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":0,\"text\":{\"format\":{\"type\":\"json_schema\",\"description\":null,\"name\":\"response\",\"schema\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":true},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":38,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":44},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" ], "kind": "sse" }, @@ -2874,12 +2874,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc965d3ec2b6-VIE", + "cf-ray": "9fdb1627cc70a0c4-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:39:22 GMT", + "date": "Mon, 18 May 2026 13:07:44 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "98", + "openai-processing-ms": "148", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -2887,7 +2887,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_940aa3b440584c3f9ca0986ddbd8be53" + "x-request-id": "req_89b403bc603e4583847de662c103ffbc" }, "status": 200, "statusText": "OK" @@ -2897,7 +2897,7 @@ "callIndex": 13, "id": "eebc152ba05b8412", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:23.847Z", + "recordedAt": "2026-05-18T13:07:45.623Z", "request": { "body": { "kind": "json", @@ -2929,11 +2929,11 @@ "billing": { "payer": "developer" }, - "completed_at": 1778845163, - "created_at": 1778845163, + "completed_at": 1779109665, + "created_at": 1779109664, "error": null, "frequency_penalty": 0, - "id": "resp_0d727e0edb39b22c006a0705eb244481979a804f4e5dbff9a1", + "id": "resp_06076d121997aaea006a0b0f20dfd481968d1ea0c85287fb38", "incomplete_details": null, "instructions": null, "max_output_tokens": 24, @@ -2952,7 +2952,7 @@ "type": "output_text" } ], - "id": "msg_0d727e0edb39b22c006a0705eba16c8197a86455c2b5558219", + "id": "msg_06076d121997aaea006a0b0f2138a88196bcd03025c07b05ba", "role": "assistant", "status": "completed", "type": "message" @@ -3001,13 +3001,13 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dc9cfcfac2b6-VIE", + "cf-ray": "9fdb162cdeeba0c4-VIE", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", - "date": "Fri, 15 May 2026 11:39:23 GMT", + "date": "Mon, 18 May 2026 13:07:45 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "705", + "openai-processing-ms": "752", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -3021,7 +3021,7 @@ "x-ratelimit-remaining-tokens": "149999965", "x-ratelimit-reset-requests": "2ms", "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_e7a949966a984192be07b061c78f9e34" + "x-request-id": "req_a62cf56f3337428bab7ebbb48d49089e" }, "status": 200, "statusText": "OK" @@ -3031,7 +3031,7 @@ "callIndex": 14, "id": "bc08e707f2920073", "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-15T11:39:24.640Z", + "recordedAt": "2026-05-18T13:07:46.742Z", "request": { "body": { "kind": "json", @@ -3059,17 +3059,17 @@ "response": { "body": { "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0612209c4b7d26aa006a0705ec0d6481978798db1fc6746893\",\"object\":\"response\",\"created_at\":1778845164,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0612209c4b7d26aa006a0705ec0d6481978798db1fc6746893\",\"object\":\"response\",\"created_at\":1778845164,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"STREAM\",\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"logprobs\":[],\"obfuscation\":\"OkK3SNpNlP\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" HEL\",\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"logprobs\":[],\"obfuscation\":\"wD3M8M0NgdEd\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"LO\",\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"logprobs\":[],\"obfuscation\":\"R6FufpH6o2XjNv\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"STREAM HELLO\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0612209c4b7d26aa006a0705ec0d6481978798db1fc6746893\",\"object\":\"response\",\"created_at\":1778845164,\"status\":\"completed\",\"background\":false,\"completed_at\":1778845164,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0612209c4b7d26aa006a0705ec6fa48197a09483c805100601\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":17,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":21},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0d44771906d507ad006a0b0f21ccb081909e644a5ec5d661c6\",\"object\":\"response\",\"created_at\":1779109665,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", + "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0d44771906d507ad006a0b0f21ccb081909e644a5ec5d661c6\",\"object\":\"response\",\"created_at\":1779109665,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"STREAM\",\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"logprobs\":[],\"obfuscation\":\"ZuEEPDw1FZ\",\"output_index\":0,\"sequence_number\":4}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" HEL\",\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"logprobs\":[],\"obfuscation\":\"C80khomvIfEM\",\"output_index\":0,\"sequence_number\":5}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"LO\",\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"logprobs\":[],\"obfuscation\":\"jhH9D3rIqFDDS2\",\"output_index\":0,\"sequence_number\":6}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"STREAM HELLO\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"},\"sequence_number\":8}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0d44771906d507ad006a0b0f21ccb081909e644a5ec5d661c6\",\"object\":\"response\",\"created_at\":1779109665,\"status\":\"completed\",\"background\":false,\"completed_at\":1779109666,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":24,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0d44771906d507ad006a0b0f2275ac81908113e861b8ac0838\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"STREAM HELLO\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":true,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":17,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":21},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" ], "kind": "sse" }, @@ -3077,12 +3077,12 @@ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", - "cf-ray": "9fc1dca29b7fc2b6-VIE", + "cf-ray": "9fdb1632ab83a0c4-VIE", "connection": "keep-alive", "content-type": "text/event-stream; charset=utf-8", - "date": "Fri, 15 May 2026 11:39:24 GMT", + "date": "Mon, 18 May 2026 13:07:45 GMT", "openai-organization": "braintrust-data", - "openai-processing-ms": "111", + "openai-processing-ms": "93", "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", "openai-version": "2020-10-01", "server": "cloudflare", @@ -3090,7 +3090,7 @@ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", - "x-request-id": "req_4bf9950c934341bd86432167ea397589" + "x-request-id": "req_a273f7cd9d58413e97aacd0145cf9122" }, "status": 200, "statusText": "OK" diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.json new file mode 100644 index 000000000..5455e638d --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.json @@ -0,0 +1,956 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 24, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "PARIS", + "usage": { + "completionTokens": 2, + "promptTokens": 18 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 18 + } + } + ], + "input": { + "maxTokens": 24, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "isContinued": false, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "stepType": "initial", + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "rawResponse": { + "headers": "" + }, + "text": "One, two, three.", + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "modelId": "text-embedding-3-small", + "settings": {}, + "specificationVersion": "v1" + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 128, + "mode": { + "toolChoice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "type": "function" + } + ], + "type": "regular" + }, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"location\":\"Paris, France\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 87 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + {} + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxTokens": 128, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "tool-calls", + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "", + "type": "text" + }, + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "tool-calls", + "isContinued": false, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "", + "type": "text" + }, + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "stepType": "initial", + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json" + } + ], + "usage": { + "completionTokens": 5, + "promptTokens": 59 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 59 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "completionTokens": 5, + "promptTokens": 59, + "totalTokens": 64 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "3.4.33", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.txt new file mode 100644 index 000000000..9cf6df0f3 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-auto-hook.span-tree.txt @@ -0,0 +1,871 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "3.4.33", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 24, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "isContinued": false, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "stepType": "initial", + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 24, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "PARIS", + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 18 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "text": "One, two, three.", + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "completionTokens": null, + │ "promptTokens": null + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": null, + │ "prompt_tokens": null, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "modelId": "text-embedding-3-small", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 128, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "", + │ "type": "text" + │ }, + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "isContinued": false, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "", + │ "type": "text" + │ }, + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "stepType": "initial", + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "inputFormat": "prompt", + │ │ "maxRetries": 2, + │ │ "maxTokens": 128, + │ │ "mode": { + │ │ "toolChoice": { + │ │ "type": "auto" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "name": "get_weather", + │ │ "parameters": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "type": "function" + │ │ } + │ │ ], + │ │ "type": "regular" + │ │ }, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "reasoningTokens": 0 + │ │ } + │ │ }, + │ │ "rawResponse": { + │ │ "headers": "" + │ │ }, + │ │ "response": { + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "toolCalls": [ + │ │ { + │ │ "args": "{\"location\":\"Paris, France\"}", + │ │ "toolCallId": "", + │ │ "toolCallType": "function", + │ │ "toolName": "get_weather" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "completionTokens": 16, + │ │ "promptTokens": 87 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.chat" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 16, + │ │ "prompt_tokens": 87 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ {} + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59, + │ "totalTokens": 64 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 32, + │ "mode": { + │ "tool": { + │ "description": "Respond with a JSON object.", + │ "name": "json", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "function" + │ }, + │ "type": "object-tool" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "toolCalls": [ + │ { + │ "args": "{\"city\":\"Paris\"}", + │ "toolCallId": "", + │ "toolCallType": "function", + │ "toolName": "json" + │ } + │ ], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 59 + │ } + └── ai-sdk-stream-object-operation + metadata: { + "operation": "stream-object", + "testRunId": "" + } + └── streamObject [function] + input: { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + } + output: { + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "warnings": [] + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + } + output: { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.json new file mode 100644 index 000000000..5455e638d --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.json @@ -0,0 +1,956 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 24, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "PARIS", + "usage": { + "completionTokens": 2, + "promptTokens": 18 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 18 + } + } + ], + "input": { + "maxTokens": 24, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "isContinued": false, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "stepType": "initial", + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "rawResponse": { + "headers": "" + }, + "text": "One, two, three.", + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "modelId": "text-embedding-3-small", + "settings": {}, + "specificationVersion": "v1" + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 128, + "mode": { + "toolChoice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "type": "function" + } + ], + "type": "regular" + }, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"location\":\"Paris, France\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 87 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + {} + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxTokens": 128, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "tool-calls", + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "", + "type": "text" + }, + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "responseMessages": "", + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "tool-calls", + "isContinued": false, + "response": { + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "", + "type": "text" + }, + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "stepType": "initial", + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "rawResponse": { + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json" + } + ], + "usage": { + "completionTokens": 5, + "promptTokens": 59 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 59 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "reasoningTokens": 0 + } + }, + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "response": { + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "completionTokens": 5, + "promptTokens": 59, + "totalTokens": 64 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "3.4.33", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.txt new file mode 100644 index 000000000..9cf6df0f3 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3-wrapped.span-tree.txt @@ -0,0 +1,871 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "3.4.33", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 24, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "isContinued": false, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "stepType": "initial", + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 24, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "PARIS", + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 18 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "text": "One, two, three.", + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "completionTokens": null, + │ "promptTokens": null + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": null, + │ "prompt_tokens": null, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "modelId": "text-embedding-3-small", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 128, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "", + │ "type": "text" + │ }, + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "responseMessages": "", + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "isContinued": false, + │ "response": { + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "", + │ "type": "text" + │ }, + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "stepType": "initial", + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "inputFormat": "prompt", + │ │ "maxRetries": 2, + │ │ "maxTokens": 128, + │ │ "mode": { + │ │ "toolChoice": { + │ │ "type": "auto" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "name": "get_weather", + │ │ "parameters": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "type": "function" + │ │ } + │ │ ], + │ │ "type": "regular" + │ │ }, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "reasoningTokens": 0 + │ │ } + │ │ }, + │ │ "rawResponse": { + │ │ "headers": "" + │ │ }, + │ │ "response": { + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "toolCalls": [ + │ │ { + │ │ "args": "{\"location\":\"Paris, France\"}", + │ │ "toolCallId": "", + │ │ "toolCallType": "function", + │ │ "toolName": "get_weather" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "completionTokens": 16, + │ │ "promptTokens": 87 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.chat" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 16, + │ │ "prompt_tokens": 87 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ {} + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59, + │ "totalTokens": 64 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxRetries": 2, + │ "maxTokens": 32, + │ "mode": { + │ "tool": { + │ "description": "Respond with a JSON object.", + │ "name": "json", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "function" + │ }, + │ "type": "object-tool" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "reasoningTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "toolCalls": [ + │ { + │ "args": "{\"city\":\"Paris\"}", + │ "toolCallId": "", + │ "toolCallType": "function", + │ "toolName": "json" + │ } + │ ], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 59 + │ } + └── ai-sdk-stream-object-operation + metadata: { + "operation": "stream-object", + "testRunId": "" + } + └── streamObject [function] + input: { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + } + output: { + "object": { + "city": "Paris" + }, + "rawResponse": { + "headers": "" + }, + "warnings": [] + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "inputFormat": "prompt", + "maxRetries": 2, + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + } + output: { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.log-payloads.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.log-payloads.json deleted file mode 100644 index b88ab7c99..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.log-payloads.json +++ /dev/null @@ -1,135 +0,0 @@ -[ - { - "input": null, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "output": null - }, - { - "input": null, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": [ - { - "location": "Paris, France" - }, - {} - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": null, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "output": {} - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.span-events.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.span-events.json deleted file mode 100644 index 039bc86a6..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v3.span-events.json +++ /dev/null @@ -1,176 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.json new file mode 100644 index 000000000..db7a47c5d --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.json @@ -0,0 +1,1390 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 24, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "PARIS", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 18, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 20 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "PARIS", + "usage": { + "completionTokens": 2, + "promptTokens": 18 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 18 + } + } + ], + "input": { + "maxTokens": 24, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "stop", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": "JSON schema:\n{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}},\"required\":[\"answer\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}\nYou MUST answer with a JSON object that matches the JSON schema above.", + "role": "system" + }, + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":\"4\"}", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 83, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 88 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "{\"answer\":\"4\"}", + "usage": { + "completionTokens": 5, + "promptTokens": 83 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 83 + } + } + ], + "input": { + "experimental_output": { + "type": "object" + }, + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "stop", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 5, + "promptTokens": 83, + "totalTokens": 88 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 5, + "promptTokens": 83, + "totalTokens": 88 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "modelId": "text-embedding-3-small", + "settings": {}, + "specificationVersion": "v1" + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 128, + "mode": { + "toolChoice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "type": "function" + } + ], + "type": "regular" + }, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 16, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 87, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 103 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"location\":\"Paris, France\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 87 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxTokens": 128, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "id": "", + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "id": "", + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "tool-calls", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "id": "", + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "id": "", + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "json" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 59, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 64 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json" + } + ], + "usage": { + "completionTokens": 5, + "promptTokens": 59 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 59 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "completionTokens": 5, + "promptTokens": 59, + "totalTokens": 64 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "4.3.19", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.txt new file mode 100644 index 000000000..f46eabdf3 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-auto-hook.span-tree.txt @@ -0,0 +1,1292 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "4.3.19", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 24, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "stop", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 24, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "PARIS", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 2, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 18, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 20 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "PARIS", + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 18 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "type": "object" + │ }, + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "stop", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83, + │ "totalTokens": 88 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83, + │ "totalTokens": 88 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": "JSON schema:\n{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}},\"required\":[\"answer\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}\nYou MUST answer with a JSON object that matches the JSON schema above.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":\"4\"}", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 5, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 83, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 88 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 83 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "completionTokens": null, + │ "promptTokens": null + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": null, + │ "prompt_tokens": null, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "modelId": "text-embedding-3-small", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 128, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "id": "", + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "tool-calls", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "id": "", + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "inputFormat": "prompt", + │ │ "maxTokens": 128, + │ │ "mode": { + │ │ "toolChoice": { + │ │ "type": "auto" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "name": "get_weather", + │ │ "parameters": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "type": "function" + │ │ } + │ │ ], + │ │ "type": "regular" + │ │ }, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "acceptedPredictionTokens": 0, + │ │ "reasoningTokens": 0, + │ │ "rejectedPredictionTokens": 0 + │ │ } + │ │ }, + │ │ "rawResponse": { + │ │ "body": { + │ │ "choices": [ + │ │ { + │ │ "finish_reason": "tool_calls", + │ │ "index": 0, + │ │ "logprobs": null, + │ │ "message": { + │ │ "annotations": [], + │ │ "content": null, + │ │ "refusal": null, + │ │ "role": "assistant", + │ │ "tool_calls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\":\"Paris, France\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ], + │ │ "created": 0, + │ │ "id": "", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "object": "chat.completion", + │ │ "service_tier": "default", + │ │ "system_fingerprint": "", + │ │ "usage": { + │ │ "completion_tokens": 16, + │ │ "completion_tokens_details": { + │ │ "accepted_prediction_tokens": 0, + │ │ "audio_tokens": 0, + │ │ "reasoning_tokens": 0, + │ │ "rejected_prediction_tokens": 0 + │ │ }, + │ │ "prompt_tokens": 87, + │ │ "prompt_tokens_details": { + │ │ "audio_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "total_tokens": 103 + │ │ } + │ │ }, + │ │ "headers": "" + │ │ }, + │ │ "response": { + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "toolCalls": [ + │ │ { + │ │ "args": "{\"location\":\"Paris, France\"}", + │ │ "toolCallId": "", + │ │ "toolCallType": "function", + │ │ "toolName": "get_weather" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "completionTokens": 16, + │ │ "promptTokens": 87 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.chat" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 16, + │ │ "prompt_tokens": 87 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59, + │ "totalTokens": 64 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "tool": { + │ "description": "Respond with a JSON object.", + │ "name": "json", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "function" + │ }, + │ "type": "object-tool" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "json" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 5, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 59, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 64 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "toolCalls": [ + │ { + │ "args": "{\"city\":\"Paris\"}", + │ "toolCallId": "", + │ "toolCallType": "function", + │ "toolName": "json" + │ } + │ ], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 59 + │ } + └── ai-sdk-stream-object-operation + metadata: { + "operation": "stream-object", + "testRunId": "" + } + └── streamObject [function] + input: { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + } + output: { + "object": { + "city": "Paris" + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + } + output: { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.json new file mode 100644 index 000000000..db7a47c5d --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.json @@ -0,0 +1,1390 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 24, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "PARIS", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 18, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 20 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "PARIS", + "usage": { + "completionTokens": 2, + "promptTokens": 18 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 18 + } + } + ], + "input": { + "maxTokens": 24, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "stop", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "PARIS", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": "JSON schema:\n{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}},\"required\":[\"answer\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}\nYou MUST answer with a JSON object that matches the JSON schema above.", + "role": "system" + }, + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":\"4\"}", + "refusal": null, + "role": "assistant" + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 83, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 88 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "text": "{\"answer\":\"4\"}", + "usage": { + "completionTokens": 5, + "promptTokens": 83 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 83 + } + } + ], + "input": { + "experimental_output": { + "type": "object" + }, + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "stop", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "id": "", + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 5, + "promptTokens": 83, + "totalTokens": 88 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "usage": { + "completionTokens": 5, + "promptTokens": 83, + "totalTokens": 88 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "type": "regular" + }, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "modelId": "text-embedding-3-small", + "settings": {}, + "specificationVersion": "v1" + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 128, + "mode": { + "toolChoice": { + "type": "auto" + }, + "tools": [ + { + "description": "Get the weather for a location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "type": "function" + } + ], + "type": "regular" + }, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 16, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 87, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 103 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"location\":\"Paris, France\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "get_weather" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 87 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxTokens": 128, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "id": "", + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "id": "", + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "files": [], + "finishReason": "tool-calls", + "isContinued": false, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "reasoningDetails": [], + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "id": "", + "role": "assistant" + }, + { + "content": [ + { + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "id": "", + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "sources": [], + "stepType": "initial", + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "args": { + "location": "Paris, France" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "args": { + "location": "Paris, France" + }, + "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "usage": { + "completionTokens": 16, + "promptTokens": 87, + "totalTokens": 103 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "rawResponse": { + "body": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": null, + "refusal": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "json" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "created": 0, + "id": "", + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 5, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 59, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 64 + } + }, + "headers": "" + }, + "response": { + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json" + } + ], + "usage": { + "completionTokens": 5, + "promptTokens": 59 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 59 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "experimental_providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "acceptedPredictionTokens": 0, + "reasoningTokens": 0, + "rejectedPredictionTokens": 0 + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "completionTokens": 5, + "promptTokens": 59, + "totalTokens": 64 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "4.3.19", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.txt new file mode 100644 index 000000000..f46eabdf3 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4-wrapped.span-tree.txt @@ -0,0 +1,1292 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "4.3.19", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 24, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "stop", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 24, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "PARIS", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 2, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 18, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 20 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "PARIS", + │ "usage": { + │ "completionTokens": 2, + │ "promptTokens": 18 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 18 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "type": "object" + │ }, + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "stop", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83, + │ "totalTokens": 88 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83, + │ "totalTokens": 88 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": "JSON schema:\n{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}},\"required\":[\"answer\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}\nYou MUST answer with a JSON object that matches the JSON schema above.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":\"4\"}", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 5, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 83, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 88 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 83 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 83 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "type": "regular" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "completionTokens": null, + │ "promptTokens": null + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": null, + │ "prompt_tokens": null, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "modelId": "text-embedding-3-small", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxTokens": 128, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "id": "", + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "files": [], + │ "finishReason": "tool-calls", + │ "isContinued": false, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "reasoningDetails": [], + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "id": "", + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "id": "", + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "sources": [], + │ "stepType": "initial", + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "result": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "usage": { + │ "completionTokens": 16, + │ "promptTokens": 87, + │ "totalTokens": 103 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "inputFormat": "prompt", + │ │ "maxTokens": 128, + │ │ "mode": { + │ │ "toolChoice": { + │ │ "type": "auto" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "name": "get_weather", + │ │ "parameters": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "type": "function" + │ │ } + │ │ ], + │ │ "type": "regular" + │ │ }, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "acceptedPredictionTokens": 0, + │ │ "reasoningTokens": 0, + │ │ "rejectedPredictionTokens": 0 + │ │ } + │ │ }, + │ │ "rawResponse": { + │ │ "body": { + │ │ "choices": [ + │ │ { + │ │ "finish_reason": "tool_calls", + │ │ "index": 0, + │ │ "logprobs": null, + │ │ "message": { + │ │ "annotations": [], + │ │ "content": null, + │ │ "refusal": null, + │ │ "role": "assistant", + │ │ "tool_calls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\":\"Paris, France\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ], + │ │ "created": 0, + │ │ "id": "", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "object": "chat.completion", + │ │ "service_tier": "default", + │ │ "system_fingerprint": "", + │ │ "usage": { + │ │ "completion_tokens": 16, + │ │ "completion_tokens_details": { + │ │ "accepted_prediction_tokens": 0, + │ │ "audio_tokens": 0, + │ │ "reasoning_tokens": 0, + │ │ "rejected_prediction_tokens": 0 + │ │ }, + │ │ "prompt_tokens": 87, + │ │ "prompt_tokens_details": { + │ │ "audio_tokens": 0, + │ │ "cached_tokens": 0 + │ │ }, + │ │ "total_tokens": 103 + │ │ } + │ │ }, + │ │ "headers": "" + │ │ }, + │ │ "response": { + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "toolCalls": [ + │ │ { + │ │ "args": "{\"location\":\"Paris, France\"}", + │ │ "toolCallId": "", + │ │ "toolCallType": "function", + │ │ "toolName": "get_weather" + │ │ } + │ │ ], + │ │ "usage": { + │ │ "completionTokens": 16, + │ │ "promptTokens": 87 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.chat" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 16, + │ │ "prompt_tokens": 87 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxTokens": 32, + │ "model": { + │ "config": { + │ "compatibility": "compatible", + │ "provider": "openai.chat" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "settings": {}, + │ "specificationVersion": "v1" + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "experimental_providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59, + │ "totalTokens": 64 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ └── doGenerate [llm] + │ input: { + │ "inputFormat": "prompt", + │ "maxTokens": 32, + │ "mode": { + │ "tool": { + │ "description": "Respond with a JSON object.", + │ "name": "json", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "function" + │ }, + │ "type": "object-tool" + │ }, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "acceptedPredictionTokens": 0, + │ "reasoningTokens": 0, + │ "rejectedPredictionTokens": 0 + │ } + │ }, + │ "rawResponse": { + │ "body": { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": null, + │ "refusal": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "json" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ], + │ "created": 0, + │ "id": "", + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "chat.completion", + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 5, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 59, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 64 + │ } + │ }, + │ "headers": "" + │ }, + │ "response": { + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "toolCalls": [ + │ { + │ "args": "{\"city\":\"Paris\"}", + │ "toolCallId": "", + │ "toolCallType": "function", + │ "toolName": "json" + │ } + │ ], + │ "usage": { + │ "completionTokens": 5, + │ "promptTokens": 59 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.chat" + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 59 + │ } + └── ai-sdk-stream-object-operation + metadata: { + "operation": "stream-object", + "testRunId": "" + } + └── streamObject [function] + input: { + "maxTokens": 32, + "model": { + "config": { + "compatibility": "compatible", + "provider": "openai.chat" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "settings": {}, + "specificationVersion": "v1" + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + } + output: { + "object": { + "city": "Paris" + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "inputFormat": "prompt", + "maxTokens": 32, + "mode": { + "tool": { + "description": "Respond with a JSON object.", + "name": "json", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "function" + }, + "type": "object-tool" + }, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + } + output: { + "finishReason": "stop", + "text": "", + "toolCalls": [ + { + "args": "{\"city\":\"Paris\"}", + "toolCallId": "", + "toolCallType": "function", + "toolName": "json", + "type": "tool-call" + } + ], + "usage": { + "completionTokens": null, + "promptTokens": null + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.chat" + } + metrics: { + "completion_tokens": null, + "prompt_tokens": null, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.log-payloads.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.log-payloads.json deleted file mode 100644 index f7cc98ef9..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.log-payloads.json +++ /dev/null @@ -1,143 +0,0 @@ -[ - { - "input": null, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "output": null - }, - { - "input": null, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": null, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "output": {} - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.span-events.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.span-events.json deleted file mode 100644 index 039bc86a6..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v4.span-events.json +++ /dev/null @@ -1,176 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": null, - "name": "generateObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.chat" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json new file mode 100644 index 000000000..5e158a49c --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.json @@ -0,0 +1,5185 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "reasoning_tokens": 0, + "tokens": 51 + } + } + ], + "input": { + "experimental_output": { + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 22, + "outputTokens": 7, + "reasoningTokens": 0, + "totalTokens": 29 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 7, + "prompt_cached_tokens": 0, + "prompt_tokens": 22, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 29 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "maxEmbeddingsPerCall": 2048, + "modelId": "text-embedding-3-small", + "specificationVersion": "v2", + "supportsParallelCalls": true + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 94, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 94, + "reasoning_tokens": 0, + "tokens": 102 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 133, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "reasoning_tokens": 0, + "tokens": 141 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 172, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 172, + "reasoning_tokens": 0, + "tokens": 180 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 211, + "reasoning_tokens": 0, + "tokens": 219 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxOutputTokens": 128, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "toolChoice": "required", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 94, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 133, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 172, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 610, + "outputTokens": 32, + "reasoningTokens": 0, + "totalTokens": 642 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0, + "tokens": 3620 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "ai-sdk-openai-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0, + "tokens": 3620 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0, + "tokens": 3620 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "openai-cache", + "testRunId": "" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 3, + "tokens": 10 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v2" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "ai-sdk-anthropic-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 3, + "tokens": 10 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v2" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 3, + "tokens": 10 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v2" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + } + ], + "metadata": { + "operation": "anthropic-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-deny-output-override-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "tokens": 20 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the word DENIED and nothing else.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "deny-output-override", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"city\":\"Paris\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "tokens": 44 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "{\"city\":\"Paris\"}", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-generate-operation", + "children": [ + { + "name": "Agent.generate", + "type": "function", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 26, + "reasoning_tokens": 0, + "tokens": 29 + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 26, + "reasoning_tokens": 0, + "tokens": 29 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "agent-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-stream-operation", + "children": [ + { + "name": "Agent.stream", + "type": "function", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + }, + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 27, + "outputTokens": 4, + "reasoningTokens": 0, + "totalTokens": 31 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 27, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 31 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ] + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "agent-stream", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "5.0.82", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt new file mode 100644 index 000000000..d0276f770 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-auto-hook.span-tree.txt @@ -0,0 +1,4954 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "5.0.82", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "reasoning_tokens": 0, + │ "tokens": 51 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 22, + │ "outputTokens": 7, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 22, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 29 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "maxEmbeddingsPerCall": 2048, + │ "modelId": "text-embedding-3-small", + │ "specificationVersion": "v2", + │ "supportsParallelCalls": true + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 128, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "toolChoice": "required", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 94, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 102 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 133, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 141 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 172, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 180 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 211, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 610, + │ "outputTokens": 32, + │ "reasoningTokens": 0, + │ "totalTokens": 642 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 211, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 94, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 102 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 94, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 102 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 133, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 141 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 133, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 141 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 172, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 180 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 172, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 180 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 211, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 219 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 211, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 219 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0, + │ "tokens": 3620 + │ } + ├── ai-sdk-openai-cache-operation + │ metadata: { + │ "operation": "openai-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v2", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ }, + │ │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "stop", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 3456, + │ │ "prompt_tokens": 3617, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 3620 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0, + │ "tokens": 3620 + │ } + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v2" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 3, + │ "tokens": 10 + │ } + ├── ai-sdk-anthropic-cache-operation + │ metadata: { + │ "operation": "anthropic-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "baseURL": "/anthropic/v1", + │ │ "provider": "anthropic.messages" + │ │ }, + │ │ "modelId": "claude-haiku-4-5", + │ │ "specificationVersion": "v2" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "claude-haiku-4-5-20251001" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "stop", + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 4516, + │ │ "prompt_tokens": 3, + │ │ "tokens": 10 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v2" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 3, + │ "tokens": 10 + │ } + ├── ai-sdk-deny-output-override-operation + │ metadata: { + │ "operation": "deny-output-override", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the word DENIED and nothing else.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "reasoning_tokens": 0, + │ "tokens": 20 + │ } + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "tokens": 44 + │ } + ├── ai-sdk-stream-object-operation + │ metadata: { + │ "operation": "stream-object", + │ "testRunId": "" + │ } + │ └── streamObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Stream JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "{\"city\":\"Paris\"}", + │ "toolCalls": [], + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── ai-sdk-agent-generate-operation + │ metadata: { + │ "operation": "agent-generate", + │ "testRunId": "" + │ } + │ └── Agent.generate [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly HELLO and no punctuation.", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "HELLO", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with exactly HELLO and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v2", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ }, + │ │ "system": "You are a terse assistant." + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 26, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 29 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "HELLO", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 26, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 29 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 26, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 29 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 26, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 29 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": "You are a terse assistant.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly HELLO and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 26, + │ "reasoning_tokens": 0, + │ "tokens": 29 + │ } + └── ai-sdk-agent-stream-operation + metadata: { + "operation": "agent-stream", + "testRunId": "" + } + └── Agent.stream [function] + input: { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ] + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO" + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "time_to_first_token": 0 + } + ├── streamText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly STREAM HELLO and no punctuation.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "system": "You are a terse assistant." + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "STREAM HELLO" + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + └── doStream [llm] + input: { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 27, + "outputTokens": 4, + "reasoningTokens": 0, + "totalTokens": 31 + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 27, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 31 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json new file mode 100644 index 000000000..3e7b7503d --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.json @@ -0,0 +1,4522 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0, + "tokens": 21 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 18, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "reasoning_tokens": 0, + "tokens": 51 + } + } + ], + "input": { + "experimental_output": { + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "type": "object" + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 45, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 22, + "outputTokens": 7, + "reasoningTokens": 0, + "totalTokens": 29 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 7, + "prompt_cached_tokens": 0, + "prompt_tokens": 22, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 29 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "maxEmbeddingsPerCall": 2048, + "modelId": "text-embedding-3-small", + "specificationVersion": "v2", + "supportsParallelCalls": true + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 94, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 94, + "reasoning_tokens": 0, + "tokens": 102 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 133, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "reasoning_tokens": 0, + "tokens": 141 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 172, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 172, + "reasoning_tokens": 0, + "tokens": 180 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "tool-calls", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 211, + "reasoning_tokens": 0, + "tokens": 219 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxOutputTokens": 128, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "toolChoice": "required", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 94, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 133, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 172, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 610, + "outputTokens": 32, + "reasoningTokens": 0, + "totalTokens": 642 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 211, + "outputTokens": 8, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-openai-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0, + "tokens": 3620 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0, + "tokens": 3620 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokens": 3617, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "openai-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-anthropic-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 3, + "tokens": 10 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v2" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 3, + "tokens": 10 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v2" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokens": 3, + "outputTokens": 7, + "totalTokens": 10 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + } + ], + "metadata": { + "operation": "anthropic-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-deny-output-override-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "tokens": 20 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the word DENIED and nothing else.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 17, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "deny-output-override", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"city\":\"Paris\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "tokens": 44 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "{\"city\":\"Paris\"}", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 38, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 44 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-generate-operation", + "children": [ + { + "name": "Agent.generate", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/5.0.82" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 26, + "reasoning_tokens": 0, + "tokens": 29 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokens": 26, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 29 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "agent-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-stream-operation", + "children": [ + { + "name": "Agent.stream", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 27, + "outputTokens": 4, + "reasoningTokens": 0, + "totalTokens": 31 + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 27, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 31 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "agent-stream", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "5.0.82", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt new file mode 100644 index 000000000..e9fc82bec --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5-wrapped.span-tree.txt @@ -0,0 +1,4317 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "5.0.82", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 18, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0, + │ "tokens": 21 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "type": "object" + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 45, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "reasoning_tokens": 0, + │ "tokens": 51 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 22, + │ "outputTokens": 7, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 22, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 29 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "maxEmbeddingsPerCall": 2048, + │ "modelId": "text-embedding-3-small", + │ "specificationVersion": "v2", + │ "supportsParallelCalls": true + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 128, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "toolChoice": "required", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 94, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 102 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 133, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 141 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 172, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 180 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 211, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 610, + │ "outputTokens": 32, + │ "reasoningTokens": 0, + │ "totalTokens": 642 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 211, + │ "outputTokens": 8, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 94, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 102 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 94, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 102 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 133, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 141 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 133, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 141 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 172, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 180 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 172, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 180 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": "tool-calls", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokens": 211, + │ │ "outputTokens": 8, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 219 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "tool-calls", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 211, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 219 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-openai-cache-operation + │ metadata: { + │ "operation": "openai-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v2", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ }, + │ │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokens": 3617, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "stop", + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 3456, + │ │ "prompt_tokens": 3617, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 3620 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokens": 3617, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0, + │ "tokens": 3620 + │ } + ├── ai-sdk-anthropic-cache-operation + │ metadata: { + │ "operation": "anthropic-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "baseURL": "/anthropic/v1", + │ │ "provider": "anthropic.messages" + │ │ }, + │ │ "modelId": "claude-haiku-4-5", + │ │ "specificationVersion": "v2" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/5.0.82" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "claude-haiku-4-5-20251001" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokens": 3, + │ │ "outputTokens": 7, + │ │ "totalTokens": 10 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": "stop", + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 4516, + │ │ "prompt_tokens": 3, + │ │ "tokens": 10 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v2" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokens": 3, + │ "outputTokens": 7, + │ "totalTokens": 10 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 3, + │ "tokens": 10 + │ } + ├── ai-sdk-deny-output-override-operation + │ metadata: { + │ "operation": "deny-output-override", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the word DENIED and nothing else.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 17, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "reasoning_tokens": 0, + │ "tokens": 20 + │ } + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "tokens": 44 + │ } + ├── ai-sdk-stream-object-operation + │ metadata: { + │ "operation": "stream-object", + │ "testRunId": "" + │ } + │ └── streamObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Stream JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "{\"city\":\"Paris\"}", + │ "toolCalls": [], + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 38, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── ai-sdk-agent-generate-operation + │ metadata: { + │ "operation": "agent-generate", + │ "testRunId": "" + │ } + │ └── Agent.generate [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly HELLO and no punctuation.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v2", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "system": "You are a terse assistant." + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "HELLO", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/5.0.82" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": "You are a terse assistant.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly HELLO and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokens": 26, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 29 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": "stop", + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 26, + │ "reasoning_tokens": 0, + │ "tokens": 29 + │ } + └── ai-sdk-agent-stream-operation + metadata: { + "operation": "agent-stream", + "testRunId": "" + } + └── Agent.stream [function] + input: { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v2", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO" + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": "You are a terse assistant.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "cachedInputTokens": 0, + "inputTokens": 27, + "outputTokens": 4, + "reasoningTokens": 0, + "totalTokens": 31 + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": "stop", + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 27, + "reasoning_tokens": 0, + "time_to_first_token": 0, + "tokens": 31 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.log-payloads.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.log-payloads.json deleted file mode 100644 index 45c3f46d5..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.log-payloads.json +++ /dev/null @@ -1,447 +0,0 @@ -[ - { - "input": null, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "output": null - }, - { - "input": null, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": null, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "agent-generate" - }, - "metrics": null, - "name": "ai-sdk-agent-generate-operation", - "output": null - }, - { - "input": { - "prompt": [ - { - "role": "user" - } - ] - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "Agent.generate", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "agent-stream" - }, - "metrics": null, - "name": "ai-sdk-agent-stream-operation", - "output": null - }, - { - "input": { - "prompt": [ - { - "role": "user" - } - ] - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "Agent.stream", - "output": {} - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.span-events.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.span-events.json deleted file mode 100644 index f7bdf41da..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v5.span-events.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agent-generate" - }, - "metrics": null, - "name": "ai-sdk-agent-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "Agent.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agent-stream" - }, - "metrics": null, - "name": "ai-sdk-agent-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "Agent.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.json new file mode 100644 index 000000000..8dfdf9e66 --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.json @@ -0,0 +1,6294 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 45, + "total": 45 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "reasoning_tokens": 0 + } + } + ], + "input": { + "experimental_output": { + "responseFormat": {} + }, + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + } + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 22, + "total": 22 + }, + "outputTokens": { + "reasoning": 0, + "text": 7, + "total": 7 + }, + "raw": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 7, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 7, + "prompt_cached_tokens": 0, + "prompt_tokens": 22, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "maxEmbeddingsPerCall": 2048, + "modelId": "text-embedding-3-small", + "specificationVersion": "v3", + "supportsParallelCalls": true + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-rerank-operation", + "children": [ + { + "name": "rerank", + "type": "function", + "children": [], + "input": { + "documents": [ + "Athens is in Greece.", + "Paris is in France.", + "Lima is in Peru." + ], + "query": "Which document is about France?" + }, + "output": [ + { + "index": 1, + "relevance_score": 0.06664825 + }, + { + "index": 0, + "relevance_score": 0.015062517 + } + ], + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "document_count": 3, + "model": "rerank-v3.5", + "provider": "cohere.reranking", + "topN": 2 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 94, + "total": 94 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 94, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 94, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 133, + "total": 133 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 133, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 172, + "total": 172 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 172, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 172, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 211, + "total": 211 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 211, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxOutputTokens": 128, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "toolChoice": "required", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 94 + }, + "inputTokens": 94, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 94, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 133 + }, + "inputTokens": 133, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 133, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 172 + }, + "inputTokens": 172, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 172, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 211 + }, + "inputTokens": 211, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 610 + }, + "inputTokens": 610, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 32 + }, + "outputTokens": 32, + "reasoningTokens": 0, + "totalTokens": 642 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 211 + }, + "inputTokens": 211, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 3456, + "noCache": 161, + "total": 3617 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "ai-sdk-openai-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 3456, + "noCache": 161, + "total": 3617 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 3456, + "noCache": 161, + "total": 3617 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "openai-cache", + "testRunId": "" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "raw": "end_turn", + "unified": "stop" + }, + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "inputTokens": { + "cacheRead": 4516, + "cacheWrite": 0, + "noCache": 3, + "total": 4519 + }, + "outputTokens": { + "total": 7 + }, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "raw": "end_turn", + "unified": "stop" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 4519 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v3" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "rawFinishReason": "end_turn", + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "totalTokens": 4526 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "ai-sdk-anthropic-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "raw": "end_turn", + "unified": "stop" + }, + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "inputTokens": { + "cacheRead": 4516, + "cacheWrite": 0, + "noCache": 3, + "total": 4519 + }, + "outputTokens": { + "total": 7 + }, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "raw": "end_turn", + "unified": "stop" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 4519 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v3" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "rawFinishReason": "end_turn", + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "totalTokens": 4526 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "raw": "end_turn", + "unified": "stop" + }, + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "inputTokens": { + "cacheRead": 4516, + "cacheWrite": 0, + "noCache": 3, + "total": 4519 + }, + "outputTokens": { + "total": 7 + }, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "raw": "end_turn", + "unified": "stop" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 4519 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v3" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "rawFinishReason": "end_turn", + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "totalTokens": 4526 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + } + ], + "metadata": { + "operation": "anthropic-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-deny-output-override-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the word DENIED and nothing else.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "deny-output-override", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"city\":\"Paris\"}", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 38, + "total": 38 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 38 + }, + "inputTokens": 38, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "{\"city\":\"Paris\"}", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 38, + "total": 38 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-generate-operation", + "children": [ + { + "name": "ToolLoopAgent.generate", + "type": "function", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + } + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 16, + "reasoning_tokens": 0, + "tokens": 19 + } + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 16, + "total": 16 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 16, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "agent-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-stream-operation", + "children": [ + { + "name": "ToolLoopAgent.stream", + "type": "function", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + } + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + }, + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 4, + "total": 4 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 4, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ] + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "agent-stream", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "6.0.1", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.txt new file mode 100644 index 000000000..59016a50f --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-auto-hook.span-tree.txt @@ -0,0 +1,6055 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "6.0.1", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "responseFormat": {} + │ }, + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ } + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 45, + │ "total": 45 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 22, + │ "total": 22 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 7, + │ "total": 7 + │ }, + │ "raw": { + │ "input_tokens": 22, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 7, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 22, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "maxEmbeddingsPerCall": 2048, + │ "modelId": "text-embedding-3-small", + │ "specificationVersion": "v3", + │ "supportsParallelCalls": true + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-rerank-operation + │ metadata: { + │ "operation": "rerank", + │ "testRunId": "" + │ } + │ └── rerank [function] + │ input: { + │ "documents": [ + │ "Athens is in Greece.", + │ "Paris is in France.", + │ "Lima is in Peru." + │ ], + │ "query": "Which document is about France?" + │ } + │ output: [ + │ { + │ "index": 1, + │ "relevance_score": 0.06664825 + │ }, + │ { + │ "index": 0, + │ "relevance_score": 0.015062517 + │ } + │ ] + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "document_count": 3, + │ "model": "rerank-v3.5", + │ "provider": "cohere.reranking", + │ "topN": 2 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 128, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "toolChoice": "required", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 94 + │ }, + │ "inputTokens": 94, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 94, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 102 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 133 + │ }, + │ "inputTokens": 133, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 133, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 141 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 172 + │ }, + │ "inputTokens": 172, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 172, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 180 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 211 + │ }, + │ "inputTokens": 211, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 211, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 610 + │ }, + │ "inputTokens": 610, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 32 + │ }, + │ "outputTokens": 32, + │ "reasoningTokens": 0, + │ "totalTokens": 642 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 211 + │ }, + │ "inputTokens": 211, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 211, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 94, + │ │ "total": 94 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 94, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 94, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 133, + │ │ "total": 133 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 133, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 133, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 172, + │ │ "total": 172 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 172, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 172, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 211, + │ │ "total": 211 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 211, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 211, + │ │ "reasoning_tokens": 0 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 3456, + │ "noCache": 161, + │ "total": 3617 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-openai-cache-operation + │ metadata: { + │ "operation": "openai-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v3", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ }, + │ │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "stop" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 3456, + │ │ "noCache": 161, + │ │ "total": 3617 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 3, + │ │ "total": 3 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "stop" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 3456, + │ │ "prompt_tokens": 3617, + │ │ "reasoning_tokens": 0 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 3456, + │ "noCache": 161, + │ "total": 3617 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0 + │ } + ├── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v3" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "rawFinishReason": "end_turn", + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "totalTokens": 4526 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 4516, + │ "cacheWrite": 0, + │ "noCache": 3, + │ "total": 4519 + │ }, + │ "outputTokens": { + │ "total": 7 + │ }, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 4519 + │ } + ├── ai-sdk-anthropic-cache-operation + │ metadata: { + │ "operation": "anthropic-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "baseURL": "/anthropic/v1", + │ │ "provider": "anthropic.messages" + │ │ }, + │ │ "modelId": "claude-haiku-4-5", + │ │ "specificationVersion": "v3" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "rawFinishReason": "end_turn", + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ }, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ }, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "raw": "end_turn", + │ │ "unified": "stop" + │ │ }, + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "claude-haiku-4-5-20251001" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 4516, + │ │ "cacheWrite": 0, + │ │ "noCache": 3, + │ │ "total": 4519 + │ │ }, + │ │ "outputTokens": { + │ │ "total": 7 + │ │ }, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "raw": "end_turn", + │ │ "unified": "stop" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 4516, + │ │ "prompt_tokens": 4519 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v3" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "rawFinishReason": "end_turn", + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "totalTokens": 4526 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 4516, + │ "cacheWrite": 0, + │ "noCache": 3, + │ "total": 4519 + │ }, + │ "outputTokens": { + │ "total": 7 + │ }, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 4519 + │ } + ├── ai-sdk-deny-output-override-operation + │ metadata: { + │ "operation": "deny-output-override", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the word DENIED and nothing else.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 17, + │ "total": 17 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 38 + │ }, + │ "inputTokens": 38, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 38, + │ "total": 38 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-stream-object-operation + │ metadata: { + │ "operation": "stream-object", + │ "testRunId": "" + │ } + │ └── streamObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Stream JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "toolCalls": [], + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 38, + │ "total": 38 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-agent-generate-operation + │ metadata: { + │ "operation": "agent-generate", + │ "testRunId": "" + │ } + │ └── ToolLoopAgent.generate [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly HELLO and no punctuation.", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "HELLO", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": "Reply with exactly HELLO and no punctuation.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v3", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ } + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "HELLO", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 16 + │ │ }, + │ │ "inputTokens": 16, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 16, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 19 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "HELLO", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 16 + │ │ }, + │ │ "inputTokens": 16, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 19 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 0, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 0, + │ │ "noCacheTokens": 16 + │ │ }, + │ │ "inputTokens": 16, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 16, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 19 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 16, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 19 + │ │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly HELLO and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 16, + │ "total": 16 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16, + │ "reasoning_tokens": 0 + │ } + └── ai-sdk-agent-stream-operation + metadata: { + "operation": "agent-stream", + "testRunId": "" + } + └── ToolLoopAgent.stream [function] + input: { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ] + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO" + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "time_to_first_token": 0 + } + ├── streamText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly STREAM HELLO and no punctuation.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ } + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "STREAM HELLO" + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + └── doStream [llm] + input: { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + } + output: { + "finishReason": { + "unified": "stop" + }, + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 4, + "total": 4 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 4, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json new file mode 100644 index 000000000..08ace76ac --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.json @@ -0,0 +1,5462 @@ +{ + "span_tree": [ + { + "name": "ai-sdk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ai-sdk-generate-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the single token PARIS and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 18, + "total": 18 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the single token PARIS and no punctuation.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "PARIS", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + } + ], + "text": "PARIS", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 18 + }, + "inputTokens": 18, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 21 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-output-object-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return a short answer for 2 + 2. Keep the answer concise.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 45, + "total": 45 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "reasoning_tokens": 0 + } + } + ], + "input": { + "experimental_output": { + "responseFormat": {} + }, + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "output": { + "response_format": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ], + "type": "object" + }, + "type": "json" + } + }, + "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "{\"answer\":\"4\"}", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + } + ], + "text": "{\"answer\":\"4\"}", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 45 + }, + "inputTokens": 45, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 45, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 51 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "output-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-operation", + "children": [ + { + "name": "streamText", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Count from 1 to 3 and include the words one two three.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "One, two, three.", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 22, + "total": 22 + }, + "outputTokens": { + "reasoning": 0, + "text": 7, + "total": 7 + }, + "raw": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 7, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 7, + "prompt_cached_tokens": 0, + "prompt_tokens": 22, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Count from 1 to 3 and include the words one two three.", + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "text": "One, two, three." + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "ai-sdk-embed-operation", + "children": [ + { + "name": "embed", + "type": "function", + "children": [], + "input": { + "model": { + "config": { + "provider": "openai.embedding" + }, + "maxEmbeddingsPerCall": 2048, + "modelId": "text-embedding-3-small", + "specificationVersion": "v3", + "supportsParallelCalls": true + }, + "value": "Paris is the capital of France." + }, + "output": { + "embedding_length": 1536, + "usage": { + "tokens": 7 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "text-embedding-3-small", + "provider": "openai.embedding" + }, + "metrics": { + "tokens": 7 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ai-sdk-rerank-operation", + "children": [ + { + "name": "rerank", + "type": "function", + "children": [], + "input": { + "documents": [ + "Athens is in Greece.", + "Paris is in France.", + "Lima is in Peru." + ], + "query": "Which document is about France?" + }, + "output": [ + { + "index": 1, + "relevance_score": 0.06664825 + }, + { + "index": 0, + "relevance_score": 0.015062517 + } + ], + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "document_count": 3, + "model": "rerank-v3.5", + "provider": "cohere.reranking", + "topN": 2 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + }, + { + "name": "ai-sdk-tool-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 94, + "total": 94 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 94, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 94, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 133, + "total": 133 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 133, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 172, + "total": 172 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 172, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 172, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 128, + "prompt": [ + { + "content": "You must inspect live weather via the provided get_weather tool before responding.", + "role": "system" + }, + { + "content": [ + { + "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "temperature": 0, + "toolChoice": { + "type": "required" + }, + "tools": [ + { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather", + "type": "function" + } + ] + }, + "output": { + "content": [ + { + "input": "{\"location\":\"Paris, France\"}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "finishReason": { + "unified": "tool-calls" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 211, + "total": 211 + }, + "outputTokens": { + "reasoning": 0, + "text": 8, + "total": 8 + }, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "tool-calls" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 211, + "reasoning_tokens": 0 + } + }, + { + "name": "get_weather", + "type": "tool", + "children": [], + "input": [ + { + "location": "Paris, France" + }, + { + "messages": [ + { + "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "role": "user" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "toolCallId": "" + } + ], + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + } + ], + "input": { + "maxOutputTokens": 128, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + "system": "You must inspect live weather via the provided get_weather tool before responding.", + "temperature": 0, + "toolChoice": "required", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + }, + "output": { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 94 + }, + "inputTokens": 94, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 94, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 102 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 133 + }, + "inputTokens": 133, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 133, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 141 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 172 + }, + "inputTokens": 172, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 172, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 180 + }, + "warnings": [] + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + }, + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "finishReason": "tool-calls", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + }, + { + "content": [ + { + "input": { + "location": "Paris, France" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "output": { + "type": "text", + "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + }, + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "role": "tool" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 211 + }, + "inputTokens": 211, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [ + { + "input": { + "location": "Paris, France" + }, + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-call" + } + ], + "toolResults": [ + { + "dynamic": false, + "input": { + "location": "Paris, France" + }, + "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "toolCallId": "", + "toolName": "get_weather", + "type": "tool-result" + } + ], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 610 + }, + "inputTokens": 610, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 32 + }, + "outputTokens": 32, + "reasoningTokens": 0, + "totalTokens": 642 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 211 + }, + "inputTokens": 211, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 8 + }, + "outputTokens": 8, + "raw": { + "input_tokens": 211, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 8, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 219 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses", + "tools": { + "get_weather": { + "description": "Get the weather for a location", + "inputSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + } + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "ai-sdk-openai-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 3456, + "noCache": 161, + "total": 3617 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 3456, + "noCache": 161, + "total": 3617 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 3456, + "prompt_tokens": 3617, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "providerOptions": { + "openai": { + "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + } + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "usage": { + "cachedInputTokens": 3456, + "inputTokenDetails": { + "cacheReadTokens": 3456, + "noCacheTokens": 161 + }, + "inputTokens": 3617, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 3617, + "input_tokens_details": { + "cached_tokens": 3456 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 3620 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "openai-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-anthropic-cache-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "raw": "end_turn", + "unified": "stop" + }, + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "inputTokens": { + "cacheRead": 4516, + "cacheWrite": 0, + "noCache": 3, + "total": 4519 + }, + "outputTokens": { + "total": 7 + }, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "raw": "end_turn", + "unified": "stop" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 4519 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v3" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "rawFinishReason": "end_turn", + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "totalTokens": 4526 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + }, + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": { + "raw": "end_turn", + "unified": "stop" + }, + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "claude-haiku-4-5-20251001" + }, + "usage": { + "inputTokens": { + "cacheRead": 4516, + "cacheWrite": 0, + "noCache": 3, + "total": 4519 + }, + "outputTokens": { + "total": 7 + }, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "raw": "end_turn", + "unified": "stop" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 4516, + "prompt_tokens": 4519 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": [ + { + "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + "type": "text" + } + ], + "providerOptions": { + "anthropic": { + "cacheControl": { + "type": "ephemeral" + } + } + }, + "role": "user" + } + ], + "model": { + "config": { + "baseURL": "/anthropic/v1", + "provider": "anthropic.messages" + }, + "modelId": "claude-haiku-4-5", + "specificationVersion": "v3" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "anthropic": { + "cacheCreationInputTokens": 0, + "container": null, + "contextManagement": null, + "iterations": null, + "stopSequence": null, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + } + } + }, + "rawFinishReason": "end_turn", + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "text": "CACHE_OK", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "claude-haiku-4-5-20251001", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + } + ], + "text": "CACHE_OK", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "totalTokens": 4526 + }, + "usage": { + "cachedInputTokens": 4516, + "inputTokenDetails": { + "cacheReadTokens": 4516, + "cacheWriteTokens": 0, + "noCacheTokens": 3 + }, + "inputTokens": 4519, + "outputTokenDetails": {}, + "outputTokens": 7, + "raw": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 4516, + "inference_geo": "not_available", + "input_tokens": 3, + "output_tokens": 7, + "service_tier": "standard" + }, + "totalTokens": 4526 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "claude-haiku-4-5", + "provider": "anthropic.messages" + } + } + ], + "metadata": { + "operation": "anthropic-cache", + "testRunId": "" + } + }, + { + "name": "ai-sdk-deny-output-override-operation", + "children": [ + { + "name": "generateText", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "text" + } + ], + "role": "user" + } + ], + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Reply with the word DENIED and nothing else.", + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "request": { + "body": { + "input": [ + { + "content": [ + { + "text": "Reply with the word DENIED and nothing else.", + "type": "input_text" + } + ], + "role": "user" + } + ], + "max_output_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "temperature": 0 + } + }, + "response": { + "body": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "DENIED", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 20 + }, + "user": null + }, + "headers": { + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "alt-svc": "h3=\":443\"; ma=86400", + "cf-cache-status": "DYNAMIC", + "cf-ray": "", + "connection": "keep-alive", + "content-type": "application/json", + "date": "", + "openai-organization": "braintrust-data", + "openai-processing-ms": "", + "openai-project": "", + "openai-version": "2020-10-01", + "server": "cloudflare", + "set-cookie": "", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "transfer-encoding": "chunked", + "x-content-type-options": "nosniff", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "", + "x-ratelimit-remaining-tokens": "", + "x-ratelimit-reset-requests": "", + "x-ratelimit-reset-tokens": "", + "x-request-id": "" + }, + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "DENIED", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + } + ], + "text": "", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 17 + }, + "inputTokens": 17, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 20 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "deny-output-override", + "testRunId": "" + } + }, + { + "name": "ai-sdk-generate-object-operation", + "children": [ + { + "name": "generateObject", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Return JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "{\"city\":\"Paris\"}", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 38, + "total": 38 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Return JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 38 + }, + "inputTokens": 38, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 6 + }, + "outputTokens": 6, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 44 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "generate-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-stream-object-operation", + "children": [ + { + "name": "streamObject", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 32, + "prompt": [ + { + "content": [ + { + "text": "Stream JSON with {\"city\":\"Paris\"}.", + "type": "text" + } + ], + "role": "user" + } + ], + "responseFormat": { + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "type": "json" + }, + "temperature": 0 + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "{\"city\":\"Paris\"}", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 38, + "total": 38 + }, + "outputTokens": { + "reasoning": 0, + "text": 6, + "total": 6 + }, + "raw": { + "input_tokens": 38, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 38, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 32, + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "temperature": 0 + }, + "output": { + "finishReason": "stop", + "object": { + "city": "Paris" + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-object", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-generate-operation", + "children": [ + { + "name": "ToolLoopAgent.generate", + "type": "function", + "children": [ + { + "name": "doGenerate", + "type": "llm", + "children": [], + "input": { + "headers": { + "user-agent": "ai/6.0.1" + }, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": { + "unified": "stop" + }, + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 16, + "total": 16 + }, + "outputTokens": { + "reasoning": 0, + "text": 3, + "total": 3 + }, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 16, + "reasoning_tokens": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "steps": [ + { + "content": [ + { + "providerMetadata": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "finishReason": "stop", + "providerMetadata": { + "openai": { + "responseId": "", + "serviceTier": "default" + } + }, + "response": { + "body": "", + "headers": "", + "id": "", + "messages": [ + { + "content": [ + { + "providerOptions": { + "openai": { + "itemId": "" + } + }, + "text": "HELLO", + "type": "text" + } + ], + "role": "assistant" + } + ], + "modelId": "gpt-4o-mini-2024-07-18", + "timestamp": "" + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + } + ], + "text": "HELLO", + "toolCalls": [], + "toolResults": [], + "totalUsage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "usage": { + "cachedInputTokens": 0, + "inputTokenDetails": { + "cacheReadTokens": 0, + "noCacheTokens": 16 + }, + "inputTokens": 16, + "outputTokenDetails": { + "reasoningTokens": 0, + "textTokens": 3 + }, + "outputTokens": 3, + "raw": { + "input_tokens": 16, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 3, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "reasoningTokens": 0, + "totalTokens": 19 + }, + "warnings": [] + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + } + ], + "metadata": { + "operation": "agent-generate", + "testRunId": "" + } + }, + { + "name": "ai-sdk-agent-stream-operation", + "children": [ + { + "name": "ToolLoopAgent.stream", + "type": "function", + "children": [ + { + "name": "doStream", + "type": "llm", + "children": [], + "input": { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + }, + "output": { + "finishReason": { + "unified": "stop" + }, + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 4, + "total": 4 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 4, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } + } + ], + "input": { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + }, + "output": { + "finishReason": "stop", + "text": "STREAM HELLO" + }, + "metadata": { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "agent-stream", + "testRunId": "" + } + } + ], + "metadata": { + "aiSdkVersion": "6.0.1", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt new file mode 100644 index 000000000..6dc18059c --- /dev/null +++ b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6-wrapped.span-tree.txt @@ -0,0 +1,5249 @@ +span_tree: +└── ai-sdk-instrumentation-root [task] + metadata: { + "aiSdkVersion": "6.0.1", + "scenario": "ai-sdk-instrumentation", + "testRunId": "" + } + ├── ai-sdk-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the single token PARIS and no punctuation.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "PARIS", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 18 + │ }, + │ "inputTokens": 18, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 21 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the single token PARIS and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "PARIS", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 18, + │ "total": 18 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 18, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-output-object-operation + │ metadata: { + │ "operation": "output-object", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "experimental_output": { + │ "responseFormat": {} + │ }, + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "output": { + │ "response_format": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ } + │ }, + │ "prompt": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "{\"answer\":\"4\"}", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 45 + │ }, + │ "inputTokens": 45, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 51 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return a short answer for 2 + 2. Keep the answer concise.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "answer": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"answer\":\"4\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 45, + │ "total": 45 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 45, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── streamText [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Count from 1 to 3 and include the words one two three.", + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "text": "One, two, three." + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Count from 1 to 3 and include the words one two three.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "text": "One, two, three.", + │ "toolCalls": [], + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 22, + │ "total": 22 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 7, + │ "total": 7 + │ }, + │ "raw": { + │ "input_tokens": 22, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 7, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 7, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 22, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed [function] + │ input: { + │ "model": { + │ "config": { + │ "provider": "openai.embedding" + │ }, + │ "maxEmbeddingsPerCall": 2048, + │ "modelId": "text-embedding-3-small", + │ "specificationVersion": "v3", + │ "supportsParallelCalls": true + │ }, + │ "value": "Paris is the capital of France." + │ } + │ output: { + │ "embedding_length": 1536, + │ "usage": { + │ "tokens": 7 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "text-embedding-3-small", + │ "provider": "openai.embedding" + │ } + │ metrics: { + │ "tokens": 7 + │ } + ├── ai-sdk-rerank-operation + │ metadata: { + │ "operation": "rerank", + │ "testRunId": "" + │ } + │ └── rerank [function] + │ input: { + │ "documents": [ + │ "Athens is in Greece.", + │ "Paris is in France.", + │ "Lima is in Peru." + │ ], + │ "query": "Which document is about France?" + │ } + │ output: [ + │ { + │ "index": 1, + │ "relevance_score": 0.06664825 + │ }, + │ { + │ "index": 0, + │ "relevance_score": 0.015062517 + │ } + │ ] + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "document_count": 3, + │ "model": "rerank-v3.5", + │ "provider": "cohere.reranking", + │ "topN": 2 + │ } + ├── ai-sdk-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 128, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "system": "You must inspect live weather via the provided get_weather tool before responding.", + │ "temperature": 0, + │ "toolChoice": "required", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ output: { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 94 + │ }, + │ "inputTokens": 94, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 94, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 102 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 133 + │ }, + │ "inputTokens": 133, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 133, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 141 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 172 + │ }, + │ "inputTokens": 172, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 172, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 180 + │ }, + │ "warnings": [] + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ }, + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "finishReason": "tool-calls", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 211 + │ }, + │ "inputTokens": 211, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 211, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "toolResults": [ + │ { + │ "dynamic": false, + │ "input": { + │ "location": "Paris, France" + │ }, + │ "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}", + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 610 + │ }, + │ "inputTokens": 610, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 32 + │ }, + │ "outputTokens": 32, + │ "reasoningTokens": 0, + │ "totalTokens": 642 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 211 + │ }, + │ "inputTokens": 211, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 8 + │ }, + │ "outputTokens": 8, + │ "raw": { + │ "input_tokens": 211, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 8, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 219 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses", + │ "tools": { + │ "get_weather": { + │ "description": "Get the weather for a location", + │ "inputSchema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ } + │ } + │ } + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 94, + │ │ "total": 94 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 94, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 94, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 133, + │ │ "total": 133 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 133, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 133, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 172, + │ │ "total": 172 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 172, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 172, + │ │ "reasoning_tokens": 0 + │ │ } + │ ├── get_weather [tool] + │ │ input: [ + │ │ { + │ │ "location": "Paris, France" + │ │ }, + │ │ { + │ │ "messages": [ + │ │ { + │ │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "toolCallId": "" + │ │ } + │ │ ] + │ │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ ├── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 128, + │ │ "prompt": [ + │ │ { + │ │ "content": "You must inspect live weather via the provided get_weather tool before responding.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "input": { + │ │ "location": "Paris, France" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "output": { + │ │ "type": "text", + │ │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ │ }, + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-result" + │ │ } + │ │ ], + │ │ "role": "tool" + │ │ } + │ │ ], + │ │ "temperature": 0, + │ │ "toolChoice": { + │ │ "type": "required" + │ │ }, + │ │ "tools": [ + │ │ { + │ │ "description": "Get the weather for a location", + │ │ "inputSchema": { + │ │ "$schema": "http://json-schema.org/draft-07/schema#", + │ │ "additionalProperties": false, + │ │ "properties": { + │ │ "location": { + │ │ "description": "The city and country", + │ │ "type": "string" + │ │ } + │ │ }, + │ │ "required": [ + │ │ "location" + │ │ ], + │ │ "type": "object" + │ │ }, + │ │ "name": "get_weather", + │ │ "type": "function" + │ │ } + │ │ ] + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "input": "{\"location\":\"Paris, France\"}", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "toolCallId": "", + │ │ "toolName": "get_weather", + │ │ "type": "tool-call" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 0, + │ │ "noCache": 211, + │ │ "total": 211 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 8, + │ │ "total": 8 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 211, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 0 + │ │ }, + │ │ "output_tokens": 8, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "tool-calls" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 8, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 211, + │ │ "reasoning_tokens": 0 + │ │ } + │ └── get_weather [tool] + │ input: [ + │ { + │ "location": "Paris, France" + │ }, + │ { + │ "messages": [ + │ { + │ "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ }, + │ { + │ "content": [ + │ { + │ "input": { + │ "location": "Paris, France" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-call" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "output": { + │ "type": "text", + │ "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + │ }, + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "toolCallId": "", + │ "toolName": "get_weather", + │ "type": "tool-result" + │ } + │ ], + │ "role": "tool" + │ } + │ ], + │ "toolCallId": "" + │ } + │ ] + │ output: "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" + ├── ai-sdk-openai-cache-operation + │ metadata: { + │ "operation": "openai-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "model": { + │ │ "config": { + │ │ "fileIdPrefixes": [ + │ │ "file-" + │ │ ], + │ │ "provider": "openai.responses" + │ │ }, + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "specificationVersion": "v3", + │ │ "supportedUrls": { + │ │ "application/pdf": [ + │ │ {} + │ │ ], + │ │ "image/*": [ + │ │ {} + │ │ ] + │ │ } + │ │ }, + │ │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "providerOptions": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 3456, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 3456, + │ │ "noCacheTokens": 161 + │ │ }, + │ │ "inputTokens": 3617, + │ │ "outputTokenDetails": { + │ │ "reasoningTokens": 0, + │ │ "textTokens": 3 + │ │ }, + │ │ "outputTokens": 3, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ }, + │ │ "reasoningTokens": 0, + │ │ "totalTokens": 3620 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "openai": { + │ │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ │ } + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "itemId": "" + │ │ } + │ │ }, + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "unified": "stop" + │ │ }, + │ │ "providerMetadata": { + │ │ "openai": { + │ │ "responseId": "", + │ │ "serviceTier": "default" + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "gpt-4o-mini-2024-07-18", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 3456, + │ │ "noCache": 161, + │ │ "total": 3617 + │ │ }, + │ │ "outputTokens": { + │ │ "reasoning": 0, + │ │ "text": 3, + │ │ "total": 3 + │ │ }, + │ │ "raw": { + │ │ "input_tokens": 3617, + │ │ "input_tokens_details": { + │ │ "cached_tokens": 3456 + │ │ }, + │ │ "output_tokens": 3, + │ │ "output_tokens_details": { + │ │ "reasoning_tokens": 0 + │ │ } + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "unified": "stop" + │ │ }, + │ │ "model": "gpt-4o-mini-2024-07-18", + │ │ "provider": "openai.responses" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 3456, + │ │ "prompt_tokens": 3617, + │ │ "reasoning_tokens": 0 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "usage": { + │ "cachedInputTokens": 3456, + │ "inputTokenDetails": { + │ "cacheReadTokens": 3456, + │ "noCacheTokens": 161 + │ }, + │ "inputTokens": 3617, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 3620 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "providerOptions": { + │ "openai": { + │ "promptCacheKey": "braintrust-ai-sdk-e2e-cache" + │ } + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 3456, + │ "noCache": 161, + │ "total": 3617 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 3617, + │ "input_tokens_details": { + │ "cached_tokens": 3456 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 3456, + │ "prompt_tokens": 3617, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-anthropic-cache-operation + │ metadata: { + │ "operation": "anthropic-cache", + │ "testRunId": "" + │ } + │ ├── generateText [function] + │ │ input: { + │ │ "maxOutputTokens": 24, + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "model": { + │ │ "config": { + │ │ "baseURL": "/anthropic/v1", + │ │ "provider": "anthropic.messages" + │ │ }, + │ │ "modelId": "claude-haiku-4-5", + │ │ "specificationVersion": "v3" + │ │ }, + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "steps": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": "stop", + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "rawFinishReason": "end_turn", + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "messages": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ], + │ │ "modelId": "claude-haiku-4-5-20251001", + │ │ "timestamp": "" + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ }, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ ], + │ │ "text": "CACHE_OK", + │ │ "toolCalls": [], + │ │ "toolResults": [], + │ │ "totalUsage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "usage": { + │ │ "cachedInputTokens": 4516, + │ │ "inputTokenDetails": { + │ │ "cacheReadTokens": 4516, + │ │ "cacheWriteTokens": 0, + │ │ "noCacheTokens": 3 + │ │ }, + │ │ "inputTokens": 4519, + │ │ "outputTokenDetails": {}, + │ │ "outputTokens": 7, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ }, + │ │ "totalTokens": 4526 + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ └── doGenerate [llm] + │ │ input: { + │ │ "headers": { + │ │ "user-agent": "ai/6.0.1" + │ │ }, + │ │ "maxOutputTokens": 24, + │ │ "prompt": [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "providerOptions": { + │ │ "anthropic": { + │ │ "cacheControl": { + │ │ "type": "ephemeral" + │ │ } + │ │ } + │ │ }, + │ │ "role": "user" + │ │ } + │ │ ], + │ │ "temperature": 0 + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "CACHE_OK", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "finishReason": { + │ │ "raw": "end_turn", + │ │ "unified": "stop" + │ │ }, + │ │ "providerMetadata": { + │ │ "anthropic": { + │ │ "cacheCreationInputTokens": 0, + │ │ "container": null, + │ │ "contextManagement": null, + │ │ "iterations": null, + │ │ "stopSequence": null, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ } + │ │ }, + │ │ "response": { + │ │ "body": "", + │ │ "headers": "", + │ │ "id": "", + │ │ "modelId": "claude-haiku-4-5-20251001" + │ │ }, + │ │ "usage": { + │ │ "inputTokens": { + │ │ "cacheRead": 4516, + │ │ "cacheWrite": 0, + │ │ "noCache": 3, + │ │ "total": 4519 + │ │ }, + │ │ "outputTokens": { + │ │ "total": 7 + │ │ }, + │ │ "raw": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 0 + │ │ }, + │ │ "cache_creation_input_tokens": 0, + │ │ "cache_read_input_tokens": 4516, + │ │ "inference_geo": "not_available", + │ │ "input_tokens": 3, + │ │ "output_tokens": 7, + │ │ "service_tier": "standard" + │ │ } + │ │ }, + │ │ "warnings": [] + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "ai-sdk", + │ │ "sdk_language": "typescript" + │ │ }, + │ │ "finish_reason": { + │ │ "raw": "end_turn", + │ │ "unified": "stop" + │ │ }, + │ │ "model": "claude-haiku-4-5", + │ │ "provider": "anthropic.messages" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cache_creation_tokens": 0, + │ │ "prompt_cached_tokens": 4516, + │ │ "prompt_tokens": 4519 + │ │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "baseURL": "/anthropic/v1", + │ "provider": "anthropic.messages" + │ }, + │ "modelId": "claude-haiku-4-5", + │ "specificationVersion": "v3" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "rawFinishReason": "end_turn", + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "claude-haiku-4-5-20251001", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "CACHE_OK", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "totalTokens": 4526 + │ }, + │ "usage": { + │ "cachedInputTokens": 4516, + │ "inputTokenDetails": { + │ "cacheReadTokens": 4516, + │ "cacheWriteTokens": 0, + │ "noCacheTokens": 3 + │ }, + │ "inputTokens": 4519, + │ "outputTokenDetails": {}, + │ "outputTokens": 7, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ }, + │ "totalTokens": 4526 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix braintrust cache prefix\n\nReply with exactly CACHE_OK and nothing else.", + │ "type": "text" + │ } + │ ], + │ "providerOptions": { + │ "anthropic": { + │ "cacheControl": { + │ "type": "ephemeral" + │ } + │ } + │ }, + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "text": "CACHE_OK", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "anthropic": { + │ "cacheCreationInputTokens": 0, + │ "container": null, + │ "contextManagement": null, + │ "iterations": null, + │ "stopSequence": null, + │ "usage": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "claude-haiku-4-5-20251001" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 4516, + │ "cacheWrite": 0, + │ "noCache": 3, + │ "total": 4519 + │ }, + │ "outputTokens": { + │ "total": 7 + │ }, + │ "raw": { + │ "cache_creation": { + │ "ephemeral_1h_input_tokens": 0, + │ "ephemeral_5m_input_tokens": 0 + │ }, + │ "cache_creation_input_tokens": 0, + │ "cache_read_input_tokens": 4516, + │ "inference_geo": "not_available", + │ "input_tokens": 3, + │ "output_tokens": 7, + │ "service_tier": "standard" + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "raw": "end_turn", + │ "unified": "stop" + │ }, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic.messages" + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 4516, + │ "prompt_tokens": 4519 + │ } + ├── ai-sdk-deny-output-override-operation + │ metadata: { + │ "operation": "deny-output-override", + │ "testRunId": "" + │ } + │ └── generateText [function] + │ input: { + │ "maxOutputTokens": 24, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Reply with the word DENIED and nothing else.", + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "request": { + │ "body": { + │ "input": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "input_text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "max_output_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "temperature": 0 + │ } + │ }, + │ "response": { + │ "body": { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output": [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "DENIED", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ], + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "usage": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ }, + │ "total_tokens": 20 + │ }, + │ "user": null + │ }, + │ "headers": { + │ "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + │ "alt-svc": "h3=\":443\"; ma=86400", + │ "cf-cache-status": "DYNAMIC", + │ "cf-ray": "", + │ "connection": "keep-alive", + │ "content-type": "application/json", + │ "date": "", + │ "openai-organization": "braintrust-data", + │ "openai-processing-ms": "", + │ "openai-project": "", + │ "openai-version": "2020-10-01", + │ "server": "cloudflare", + │ "set-cookie": "", + │ "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + │ "transfer-encoding": "chunked", + │ "x-content-type-options": "nosniff", + │ "x-ratelimit-limit-requests": "30000", + │ "x-ratelimit-limit-tokens": "150000000", + │ "x-ratelimit-remaining-requests": "", + │ "x-ratelimit-remaining-tokens": "", + │ "x-ratelimit-reset-requests": "", + │ "x-ratelimit-reset-tokens": "", + │ "x-request-id": "" + │ }, + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 17 + │ }, + │ "inputTokens": 17, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 20 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with the word DENIED and nothing else.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "DENIED", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 17, + │ "total": 17 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 17, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-generate-object-operation + │ metadata: { + │ "operation": "generate-object", + │ "testRunId": "" + │ } + │ └── generateObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Return JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 38 + │ }, + │ "inputTokens": 38, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 6 + │ }, + │ "outputTokens": 6, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 44 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Return JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 38, + │ "total": 38 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0 + │ } + ├── ai-sdk-stream-object-operation + │ metadata: { + │ "operation": "stream-object", + │ "testRunId": "" + │ } + │ └── streamObject [function] + │ input: { + │ "maxOutputTokens": 32, + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "prompt": "Stream JSON with {\"city\":\"Paris\"}.", + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": "stop", + │ "object": { + │ "city": "Paris" + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + │ └── doStream [llm] + │ input: { + │ "includeRawChunks": false, + │ "maxOutputTokens": 32, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Stream JSON with {\"city\":\"Paris\"}.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "responseFormat": { + │ "schema": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "type": "json" + │ }, + │ "temperature": 0 + │ } + │ output: { + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "text": "{\"city\":\"Paris\"}", + │ "toolCalls": [], + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 38, + │ "total": 38 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 6, + │ "total": 6 + │ }, + │ "raw": { + │ "input_tokens": 38, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 6, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ } + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 38, + │ "reasoning_tokens": 0, + │ "time_to_first_token": 0 + │ } + ├── ai-sdk-agent-generate-operation + │ metadata: { + │ "operation": "agent-generate", + │ "testRunId": "" + │ } + │ └── ToolLoopAgent.generate [function] + │ input: { + │ "maxOutputTokens": 24, + │ "messages": [ + │ { + │ "content": "Reply with exactly HELLO and no punctuation.", + │ "role": "user" + │ } + │ ], + │ "model": { + │ "config": { + │ "fileIdPrefixes": [ + │ "file-" + │ ], + │ "provider": "openai.responses" + │ }, + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "specificationVersion": "v3", + │ "supportedUrls": { + │ "application/pdf": [ + │ {} + │ ], + │ "image/*": [ + │ {} + │ ] + │ } + │ }, + │ "system": "You are a terse assistant." + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "steps": [ + │ { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": "stop", + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "messages": [ + │ { + │ "content": [ + │ { + │ "providerOptions": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ], + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "warnings": [] + │ } + │ ], + │ "text": "HELLO", + │ "toolCalls": [], + │ "toolResults": [], + │ "totalUsage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "usage": { + │ "cachedInputTokens": 0, + │ "inputTokenDetails": { + │ "cacheReadTokens": 0, + │ "noCacheTokens": 16 + │ }, + │ "inputTokens": 16, + │ "outputTokenDetails": { + │ "reasoningTokens": 0, + │ "textTokens": 3 + │ }, + │ "outputTokens": 3, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ }, + │ "reasoningTokens": 0, + │ "totalTokens": 19 + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ └── doGenerate [llm] + │ input: { + │ "headers": { + │ "user-agent": "ai/6.0.1" + │ }, + │ "maxOutputTokens": 24, + │ "prompt": [ + │ { + │ "content": [ + │ { + │ "text": "Reply with exactly HELLO and no punctuation.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "content": [ + │ { + │ "providerMetadata": { + │ "openai": { + │ "itemId": "" + │ } + │ }, + │ "text": "HELLO", + │ "type": "text" + │ } + │ ], + │ "finishReason": { + │ "unified": "stop" + │ }, + │ "providerMetadata": { + │ "openai": { + │ "responseId": "", + │ "serviceTier": "default" + │ } + │ }, + │ "response": { + │ "body": "", + │ "headers": "", + │ "id": "", + │ "modelId": "gpt-4o-mini-2024-07-18", + │ "timestamp": "" + │ }, + │ "usage": { + │ "inputTokens": { + │ "cacheRead": 0, + │ "noCache": 16, + │ "total": 16 + │ }, + │ "outputTokens": { + │ "reasoning": 0, + │ "text": 3, + │ "total": 3 + │ }, + │ "raw": { + │ "input_tokens": 16, + │ "input_tokens_details": { + │ "cached_tokens": 0 + │ }, + │ "output_tokens": 3, + │ "output_tokens_details": { + │ "reasoning_tokens": 0 + │ } + │ } + │ }, + │ "warnings": [] + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "ai-sdk", + │ "sdk_language": "typescript" + │ }, + │ "finish_reason": { + │ "unified": "stop" + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai.responses" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16, + │ "reasoning_tokens": 0 + │ } + └── ai-sdk-agent-stream-operation + metadata: { + "operation": "agent-stream", + "testRunId": "" + } + └── ToolLoopAgent.stream [function] + input: { + "maxOutputTokens": 24, + "messages": [ + { + "content": "Reply with exactly STREAM HELLO and no punctuation.", + "role": "user" + } + ], + "model": { + "config": { + "fileIdPrefixes": [ + "file-" + ], + "provider": "openai.responses" + }, + "modelId": "gpt-4o-mini-2024-07-18", + "specificationVersion": "v3", + "supportedUrls": { + "application/pdf": [ + {} + ], + "image/*": [ + {} + ] + } + }, + "system": "You are a terse assistant." + } + output: { + "finishReason": "stop", + "text": "STREAM HELLO" + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "time_to_first_token": 0 + } + └── doStream [llm] + input: { + "includeRawChunks": false, + "maxOutputTokens": 24, + "prompt": [ + { + "content": [ + { + "text": "Reply with exactly STREAM HELLO and no punctuation.", + "type": "text" + } + ], + "role": "user" + } + ] + } + output: { + "finishReason": { + "unified": "stop" + }, + "text": "STREAM HELLO", + "toolCalls": [], + "usage": { + "inputTokens": { + "cacheRead": 0, + "noCache": 17, + "total": 17 + }, + "outputTokens": { + "reasoning": 0, + "text": 4, + "total": 4 + }, + "raw": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 4, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + } + metadata: { + "braintrust": { + "integration_name": "ai-sdk", + "sdk_language": "typescript" + }, + "finish_reason": { + "unified": "stop" + }, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai.responses" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "reasoning_tokens": 0, + "time_to_first_token": 0 + } diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.log-payloads.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.log-payloads.json deleted file mode 100644 index 9fef56321..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.log-payloads.json +++ /dev/null @@ -1,503 +0,0 @@ -[ - { - "input": null, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "output": null - }, - { - "input": null, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "output": null - }, - { - "input": { - "prompt": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "rerank" - }, - "metrics": null, - "name": "ai-sdk-rerank-operation", - "output": null - }, - { - "input": { - "documents": [ - "Athens is in Greece.", - "Paris is in France.", - "Lima is in Peru." - ], - "query": "Which document is about France?" - }, - "metadata": { - "model": "rerank-v3.5", - "provider": "cohere.reranking" - }, - "metrics": null, - "name": "rerank", - "output": {} - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": [ - { - "location": "Paris, France" - }, - { - "messages": [ - { - "content": "Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.", - "role": "user" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - }, - { - "content": [ - { - "input": { - "location": "Paris, France" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-call" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "output": { - "type": "text", - "value": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - "providerOptions": { - "openai": { - "itemId": "" - } - }, - "toolCallId": "", - "toolName": "get_weather", - "type": "tool-result" - } - ], - "role": "tool" - } - ], - "toolCallId": "" - } - ], - "metadata": null, - "metrics": null, - "name": "get_weather", - "output": "{\"condition\":\"sunny\",\"location\":\"Paris, France\",\"temperatureC\":22}" - }, - { - "input": null, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "output": null - }, - { - "input": { - "prompt": "", - "schema": "" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "agent-generate" - }, - "metrics": null, - "name": "ai-sdk-agent-generate-operation", - "output": null - }, - { - "input": { - "prompt": [ - { - "role": "user" - } - ] - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "ToolLoopAgent.generate", - "output": {} - }, - { - "input": null, - "metadata": { - "operation": "agent-stream" - }, - "metrics": null, - "name": "ai-sdk-agent-stream-operation", - "output": null - }, - { - "input": { - "prompt": [ - { - "role": "user" - } - ] - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "ToolLoopAgent.stream", - "output": {} - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.span-events.json b/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.span-events.json deleted file mode 100644 index 34f4f69a4..000000000 --- a/e2e/scenarios/ai-sdk-instrumentation/__snapshots__/ai-sdk-v6.span-events.json +++ /dev/null @@ -1,301 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "aiSdkVersion": "", - "scenario": "ai-sdk-instrumentation" - }, - "metrics": null, - "name": "ai-sdk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metrics": null, - "name": "ai-sdk-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": null, - "name": "ai-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metrics": null, - "name": "ai-sdk-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateText", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metrics": null, - "name": "ai-sdk-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "rerank-v3.5", - "provider": "cohere.reranking" - }, - "metrics": null, - "name": "rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metrics": null, - "name": "get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate-object" - }, - "metrics": null, - "name": "ai-sdk-generate-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "generateObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-object" - }, - "metrics": null, - "name": "ai-sdk-stream-object-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "streamObject", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agent-generate" - }, - "metrics": null, - "name": "ai-sdk-agent-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": null, - "name": "ToolLoopAgent.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agent-stream" - }, - "metrics": null, - "name": "ai-sdk-agent-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai.responses" - }, - "metrics": { - "time_to_first_token": 0 - }, - "name": "ToolLoopAgent.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/ai-sdk-instrumentation/assertions.ts b/e2e/scenarios/ai-sdk-instrumentation/assertions.ts index ff5810a73..ab0a67461 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/assertions.ts @@ -1,15 +1,16 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findAllSpans, findChildSpans, @@ -248,13 +249,6 @@ function findOutputObjectTrace(events: CapturedLogEvent[]) { ); } -function findAttachmentTrace(events: CapturedLogEvent[]) { - return findGenerateTextTraceForOperation( - events, - "ai-sdk-attachment-operation", - ); -} - function findDenyOutputOverrideTrace(events: CapturedLogEvent[]) { return findGenerateTextTraceForOperation( events, @@ -457,37 +451,6 @@ function extractFinishReason( : undefined; } -function extractFileAttachmentReference( - input: unknown, -): Record | undefined { - if (!isRecord(input) || !Array.isArray(input.messages)) { - return undefined; - } - - for (const message of input.messages) { - if (!isRecord(message) || !Array.isArray(message.content)) { - continue; - } - - for (const part of message.content) { - if (!isRecord(part) || part.type !== "file") { - continue; - } - - const data = part.data; - if (isRecord(data) && isRecord(data.reference)) { - return data.reference; - } - - if (isRecord(data) && data.type === "braintrust_attachment") { - return data; - } - } - } - - return undefined; -} - function normalizeAISDKContext(value: unknown): Json { const context = isRecord(value) ? value : {}; return { @@ -585,29 +548,6 @@ function snapshotValue(value: unknown): Json { return normalizeAISDKSnapshotValue(structuredClone(value)) as Json; } -function summarizeAISDKSpan(event: CapturedLogEvent): Json { - return { - has_input: event.input !== undefined && event.input !== null, - has_output: event.output !== undefined && event.output !== null, - metadata: pickMetadata( - event.row.metadata as Record | undefined, - ["aiSdkVersion", "provider", "model", "operation", "scenario"], - ), - metrics: pickMetrics(event.metrics, [ - "completion_tokens", - "prompt_tokens", - "prompt_cached_tokens", - "prompt_cache_creation_tokens", - "time_to_first_token", - "tokens", - ]), - name: event.span.name ?? null, - root_span_id: event.span.rootId ?? null, - span_id: event.span.id ?? null, - span_parents: event.span.parentIds, - } satisfies Json; -} - function summarizeAISDKInput(value: unknown): Json { if (!isRecord(value)) { return snapshotValue(value); @@ -707,25 +647,7 @@ function collectSummaryEvents( ].filter((event): event is CapturedLogEvent => event !== undefined); } -function buildSpanSummary( - events: CapturedLogEvent[], - options: { - agentSpanName?: AgentSpanName; - sdkMajorVersion: number; - supportsProviderCacheAssertions: boolean; - supportsGenerateObject: boolean; - supportsRerank: boolean; - supportsStreamObject: boolean; - }, -): Json { - return normalizeForSnapshot( - collectSummaryEvents(events, options).map((event) => - summarizeAISDKSpan(event), - ), - ); -} - -function buildPayloadSummary( +function buildSpanTree( events: CapturedLogEvent[], options: { agentSpanName?: AgentSpanName; @@ -735,12 +657,20 @@ function buildPayloadSummary( supportsRerank: boolean; supportsStreamObject: boolean; }, -): Json { - return normalizeForSnapshot( - collectSummaryEvents(events, options).map((event) => - summarizeAISDKPayload(event), - ), - ); +): SpanTreeEntry[] { + return collectSummaryEvents(events, options).map((event) => { + const summary = summarizeAISDKPayload(event) as Record; + const { name: _name, ...fields } = summary; + + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + }, + name: typeof summary.name === "string" ? summary.name : event.span.name, + }; + }); } function expectOperationParentedByRoot( @@ -835,7 +765,6 @@ export function defineAISDKInstrumentationAssertions(options: { runScenario: RunAISDKScenario; sdkMajorVersion: number; snapshotName: string; - supportsAttachmentScenario: boolean; supportsProviderCacheAssertions: boolean; supportsDenyOutputOverrideScenario: boolean; supportsEmbedMany: boolean; @@ -849,11 +778,7 @@ export function defineAISDKInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -1102,30 +1027,6 @@ export function defineAISDKInstrumentationAssertions(options: { ); } - if (options.supportsAttachmentScenario) { - test( - "captures file attachment normalization in input", - testConfig, - () => { - const root = findLatestSpan(events, ROOT_NAME); - const trace = findAttachmentTrace(events); - - expectOperationParentedByRoot(trace.operation, root); - expectAISDKParentSpan(trace.parent); - expect(operationName(trace.operation)).toBe("attachment"); - const attachmentRef = extractFileAttachmentReference( - trace.parent?.input, - ); - expect(attachmentRef).toBeDefined(); - expect(attachmentRef).toMatchObject({ - content_type: "text/plain", - key: expect.any(String), - type: "braintrust_attachment", - }); - }, - ); - } - test("captures trace for generateText() with tools", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const trace = findToolTrace(events); @@ -1317,38 +1218,8 @@ export function defineAISDKInstrumentationAssertions(options: { ); } - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildSpanSummary(events, { - agentSpanName: options.agentSpanName, - sdkMajorVersion: options.sdkMajorVersion, - supportsProviderCacheAssertions: - options.supportsProviderCacheAssertions, - supportsGenerateObject: options.supportsGenerateObject, - supportsRerank: options.supportsRerank, - supportsStreamObject: options.supportsStreamObject, - }), - ), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildPayloadSummary(events, { - agentSpanName: options.agentSpanName, - sdkMajorVersion: options.sdkMajorVersion, - supportsProviderCacheAssertions: - options.supportsProviderCacheAssertions, - supportsGenerateObject: options.supportsGenerateObject, - supportsRerank: options.supportsRerank, - supportsStreamObject: options.supportsStreamObject, - }), - ), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs index 58839f60a..b9057602f 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.impl.mjs @@ -456,41 +456,6 @@ async function runAISDKInstrumentationScenario( }, ); } - - if (supportsRichInputScenarios) { - await runOperation( - "ai-sdk-attachment-operation", - "attachment", - async () => { - try { - await instrumentedAI.generateText({ - model: openaiModel, - messages: [ - { - role: "user", - content: [ - { - type: "file", - data: Buffer.from("tiny test file", "utf8"), - mediaType: "text/plain", - filename: "tiny.txt", - }, - { - type: "text", - text: "Read the file and summarize it in one short sentence.", - }, - ], - }, - ], - temperature: 0, - ...tokenLimit(options.maxTokensKey, 48), - }); - } catch { - // Input attachment processing is exercised before provider errors. - } - }, - ); - } }, flushCount, flushDelayMs, diff --git a/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts b/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts index b944e6b8c..150d66501 100644 --- a/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts +++ b/e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts @@ -34,7 +34,6 @@ describe.concurrent("variants", () => { const sdkMajorVersion = parseMajorVersion(scenario.version); const supportsRichInputScenarios = sdkMajorVersion >= 5; const supportsOutputObjectScenario = supportsRichInputScenarios; - const supportsAttachmentScenario = supportsRichInputScenarios; describe.sequential(`ai sdk ${scenario.version}`, () => { defineAISDKInstrumentationAssertions({ @@ -51,8 +50,7 @@ describe.concurrent("variants", () => { timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, - supportsAttachmentScenario, + snapshotName: `${scenario.snapshotName}-wrapped`, supportsProviderCacheAssertions: scenario.supportsProviderCacheAssertions, supportsDenyOutputOverrideScenario: supportsRichInputScenarios, @@ -82,8 +80,7 @@ describe.concurrent("variants", () => { timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, - supportsAttachmentScenario, + snapshotName: `${scenario.snapshotName}-auto-hook`, supportsProviderCacheAssertions: scenario.supportsProviderCacheAssertions, supportsDenyOutputOverrideScenario: supportsRichInputScenarios, diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json deleted file mode 100644 index fd5442920..000000000 --- a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.otel-spans.json +++ /dev/null @@ -1,54 +0,0 @@ -[ - { - "hasParent": false, - "name": "ai.generateText" - }, - { - "hasParent": false, - "name": "ai.generateText" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.streamText" - }, - { - "hasParent": false, - "name": "ai.streamText.doStream" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - } -] diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.json new file mode 100644 index 000000000..a981660f5 --- /dev/null +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.json @@ -0,0 +1,392 @@ +{ + "span_tree": [ + { + "name": "ai.generateText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText", + "ai.prompt": "{\"prompt\":\"Reply with the single token PARIS and no punctuation.\"}", + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "stop", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562\",\"serviceTier\":\"default\"}}", + "ai.response.text": "PARIS", + "ai.settings.maxOutputTokens": 24, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-generate", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 3, + "ai.usage.promptTokens": 18, + "operation.name": "ai.generateText otel-generate", + "resource.name": "otel-generate" + } + }, + { + "name": "ai.generateText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText", + "ai.prompt": "{\"system\":\"You must inspect live weather via the provided get_weather tool before responding.\",\"prompt\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}", + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "tool-calls", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8\",\"serviceTier\":\"default\"}}", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_RPz7wCVGzI2qxJp9qnbZARi6\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 211, + "operation.name": "ai.generateText otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055937f0819081343d06b3562e91\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055b0f308190baaed644a23daf57\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_RPz7wCVGzI2qxJp9qnbZARi6\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 211, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 211, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055937f0819081343d06b3562e91\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 172, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 172, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 133, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 133, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 94, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 94, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Reply with the single token PARIS and no punctuation.\"}]}]", + "ai.request.headers.user-agent": "ai/5.0.82", + "ai.response.finishReason": "stop", + "ai.response.id": "resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562\",\"serviceTier\":\"default\"}}", + "ai.response.text": "PARIS", + "ai.response.timestamp": "", + "ai.settings.maxOutputTokens": 24, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-generate", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 3, + "ai.usage.promptTokens": 18, + "gen_ai.request.max_tokens": 24, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "stop" + ], + "gen_ai.response.id": "resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 18, + "gen_ai.usage.output_tokens": 3, + "operation.name": "ai.generateText.doGenerate otel-generate", + "resource.name": "otel-generate" + } + }, + { + "name": "ai.streamText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText", + "ai.prompt": "{\"prompt\":\"Count from 1 to 3 and include the words one two three.\"}", + "ai.response.finishReason": "stop", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12\",\"serviceTier\":\"default\"}}", + "ai.response.text": "One, two, three.", + "ai.settings.maxOutputTokens": 32, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-stream", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.cachedInputTokens": 0, + "ai.usage.inputTokens": 22, + "ai.usage.outputTokens": 7, + "ai.usage.reasoningTokens": 0, + "ai.usage.totalTokens": 29, + "operation.name": "ai.streamText otel-stream", + "resource.name": "otel-stream" + } + }, + { + "name": "ai.streamText.doStream", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Count from 1 to 3 and include the words one two three.\"}]}]", + "ai.response.avgOutputTokensPerSecond": 0, + "ai.response.finishReason": "stop", + "ai.response.id": "resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFinish": 0, + "ai.response.msToFirstChunk": 0, + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12\",\"serviceTier\":\"default\"}}", + "ai.response.text": "One, two, three.", + "ai.response.timestamp": "", + "ai.settings.maxOutputTokens": 32, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-stream", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "5.0.82", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.cachedInputTokens": 0, + "ai.usage.inputTokens": 22, + "ai.usage.outputTokens": 7, + "ai.usage.reasoningTokens": 0, + "ai.usage.totalTokens": 29, + "gen_ai.request.max_tokens": 32, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "stop" + ], + "gen_ai.response.id": "resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 7, + "operation.name": "ai.streamText.doStream otel-stream", + "resource.name": "otel-stream" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_pqjmElBX0Bz2RbFnYuNJUwYB", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_RPz7wCVGzI2qxJp9qnbZARi6", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_T7CMC37u6NksREpf42D604Ar", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_ykNmJqHpp2Kn5ma7KqURaYkY", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.txt b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.txt new file mode 100644 index 000000000..eab0b6cff --- /dev/null +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v5.span-tree.txt @@ -0,0 +1,350 @@ +span_tree: +├── ai.generateText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText", +│ "ai.prompt": "{\"prompt\":\"Reply with the single token PARIS and no punctuation.\"}", +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "stop", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "PARIS", +│ "ai.settings.maxOutputTokens": 24, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-generate", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 3, +│ "ai.usage.promptTokens": 18, +│ "operation.name": "ai.generateText otel-generate", +│ "resource.name": "otel-generate" +│ } +├── ai.generateText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText", +│ "ai.prompt": "{\"system\":\"You must inspect live weather via the provided get_weather tool before responding.\",\"prompt\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}", +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8\",\"serviceTier\":\"default\"}}", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_RPz7wCVGzI2qxJp9qnbZARi6\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 211, +│ "operation.name": "ai.generateText otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055937f0819081343d06b3562e91\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055b0f308190baaed644a23daf57\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_RPz7wCVGzI2qxJp9qnbZARi6\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 211, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_0cd38c0201be9bbf006a07055bac988190b27b3fadea2852c8", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 211, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a07055937f0819081343d06b3562e91\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_pqjmElBX0Bz2RbFnYuNJUwYB\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 172, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_0cd38c0201be9bbf006a070559ae1481909fa6720aa5eb2d82", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 172, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_0cd38c0201be9bbf006a070557f95481909f5a1e78dde46592\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_T7CMC37u6NksREpf42D604Ar\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 133, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_0cd38c0201be9bbf006a070558bf4881909d9f3f5439fdf72a", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 133, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_ykNmJqHpp2Kn5ma7KqURaYkY\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 94, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_0cd38c0201be9bbf006a07055606ec8190bcc0557886077291", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 94, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Reply with the single token PARIS and no punctuation.\"}]}]", +│ "ai.request.headers.user-agent": "ai/5.0.82", +│ "ai.response.finishReason": "stop", +│ "ai.response.id": "resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "PARIS", +│ "ai.response.timestamp": "", +│ "ai.settings.maxOutputTokens": 24, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-generate", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 3, +│ "ai.usage.promptTokens": 18, +│ "gen_ai.request.max_tokens": 24, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "stop" +│ ], +│ "gen_ai.response.id": "resp_000693f48a943173006a0705537bc88190b1e7fe652c78e562", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 18, +│ "gen_ai.usage.output_tokens": 3, +│ "operation.name": "ai.generateText.doGenerate otel-generate", +│ "resource.name": "otel-generate" +│ } +├── ai.streamText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.streamText", +│ "ai.prompt": "{\"prompt\":\"Count from 1 to 3 and include the words one two three.\"}", +│ "ai.response.finishReason": "stop", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "One, two, three.", +│ "ai.settings.maxOutputTokens": 32, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-stream", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.cachedInputTokens": 0, +│ "ai.usage.inputTokens": 22, +│ "ai.usage.outputTokens": 7, +│ "ai.usage.reasoningTokens": 0, +│ "ai.usage.totalTokens": 29, +│ "operation.name": "ai.streamText otel-stream", +│ "resource.name": "otel-stream" +│ } +├── ai.streamText.doStream +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.streamText.doStream", +│ "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Count from 1 to 3 and include the words one two three.\"}]}]", +│ "ai.response.avgOutputTokensPerSecond": 0, +│ "ai.response.finishReason": "stop", +│ "ai.response.id": "resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.msToFinish": 0, +│ "ai.response.msToFirstChunk": 0, +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "One, two, three.", +│ "ai.response.timestamp": "", +│ "ai.settings.maxOutputTokens": 32, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-stream", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "5.0.82", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.cachedInputTokens": 0, +│ "ai.usage.inputTokens": 22, +│ "ai.usage.outputTokens": 7, +│ "ai.usage.reasoningTokens": 0, +│ "ai.usage.totalTokens": 29, +│ "gen_ai.request.max_tokens": 32, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "stop" +│ ], +│ "gen_ai.response.id": "resp_0943012ba070f50c006a070554aebc8197a767625f31e10f12", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 22, +│ "gen_ai.usage.output_tokens": 7, +│ "operation.name": "ai.streamText.doStream otel-stream", +│ "resource.name": "otel-stream" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_pqjmElBX0Bz2RbFnYuNJUwYB", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_RPz7wCVGzI2qxJp9qnbZARi6", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_T7CMC37u6NksREpf42D604Ar", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +└── ai.toolCall + attributes: { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_ykNmJqHpp2Kn5ma7KqURaYkY", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json deleted file mode 100644 index fd5442920..000000000 --- a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.otel-spans.json +++ /dev/null @@ -1,54 +0,0 @@ -[ - { - "hasParent": false, - "name": "ai.generateText" - }, - { - "hasParent": false, - "name": "ai.generateText" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.generateText.doGenerate" - }, - { - "hasParent": false, - "name": "ai.streamText" - }, - { - "hasParent": false, - "name": "ai.streamText.doStream" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - }, - { - "hasParent": false, - "name": "ai.toolCall" - } -] diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.json b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.json new file mode 100644 index 000000000..bc9fa4416 --- /dev/null +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.json @@ -0,0 +1,392 @@ +{ + "span_tree": [ + { + "name": "ai.generateText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText", + "ai.prompt": "{\"prompt\":\"Reply with the single token PARIS and no punctuation.\"}", + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "stop", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091\",\"serviceTier\":\"default\"}}", + "ai.response.text": "PARIS", + "ai.settings.maxOutputTokens": 24, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-generate", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 3, + "ai.usage.promptTokens": 18, + "operation.name": "ai.generateText otel-generate", + "resource.name": "otel-generate" + } + }, + { + "name": "ai.generateText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText", + "ai.prompt": "{\"system\":\"You must inspect live weather via the provided get_weather tool before responding.\",\"prompt\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}", + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "tool-calls", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d\",\"serviceTier\":\"default\"}}", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_MATzU9ZSWZX95csufkA3UILm\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 211, + "operation.name": "ai.generateText otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056417788193a965dc0fa61cc243\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056417788193a965dc0fa61cc243\"}}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_MATzU9ZSWZX95csufkA3UILm\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 211, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 211, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 172, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 172, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_04450c837917b018006a0705626088819390136ae45bd37fb6", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a0705626088819390136ae45bd37fb6\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 133, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_04450c837917b018006a0705626088819390136ae45bd37fb6", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 133, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]}]", + "ai.prompt.toolChoice": "{\"type\":\"required\"}", + "ai.prompt.tools": [ + "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" + ], + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "tool-calls", + "ai.response.id": "resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49\",\"serviceTier\":\"default\"}}", + "ai.response.timestamp": "", + "ai.response.toolCalls": "[{\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", + "ai.settings.maxOutputTokens": 128, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-tool", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 8, + "ai.usage.promptTokens": 94, + "gen_ai.request.max_tokens": 128, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "tool-calls" + ], + "gen_ai.response.id": "resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 94, + "gen_ai.usage.output_tokens": 8, + "operation.name": "ai.generateText.doGenerate otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.generateText.doGenerate", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Reply with the single token PARIS and no punctuation.\"}]}]", + "ai.request.headers.user-agent": "ai/6.0.1", + "ai.response.finishReason": "stop", + "ai.response.id": "resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091\",\"serviceTier\":\"default\"}}", + "ai.response.text": "PARIS", + "ai.response.timestamp": "", + "ai.settings.maxOutputTokens": 24, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-generate", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.completionTokens": 3, + "ai.usage.promptTokens": 18, + "gen_ai.request.max_tokens": 24, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "stop" + ], + "gen_ai.response.id": "resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 18, + "gen_ai.usage.output_tokens": 3, + "operation.name": "ai.generateText.doGenerate otel-generate", + "resource.name": "otel-generate" + } + }, + { + "name": "ai.streamText", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText", + "ai.prompt": "{\"prompt\":\"Count from 1 to 3 and include the words one two three.\"}", + "ai.response.finishReason": "stop", + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60\",\"serviceTier\":\"default\"}}", + "ai.response.text": "One, two, three.", + "ai.settings.maxOutputTokens": 32, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-stream", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.cachedInputTokens": 0, + "ai.usage.inputTokens": 22, + "ai.usage.outputTokens": 7, + "ai.usage.reasoningTokens": 0, + "ai.usage.totalTokens": 29, + "operation.name": "ai.streamText otel-stream", + "resource.name": "otel-stream" + } + }, + { + "name": "ai.streamText.doStream", + "children": [], + "attributes": { + "ai.model.id": "gpt-4o-mini-2024-07-18", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Count from 1 to 3 and include the words one two three.\"}]}]", + "ai.response.avgOutputTokensPerSecond": 0, + "ai.response.finishReason": "stop", + "ai.response.id": "resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFinish": 0, + "ai.response.msToFirstChunk": 0, + "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60\",\"serviceTier\":\"default\"}}", + "ai.response.text": "One, two, three.", + "ai.response.timestamp": "", + "ai.settings.maxOutputTokens": 32, + "ai.settings.maxRetries": 2, + "ai.settings.temperature": 0, + "ai.telemetry.functionId": "otel-stream", + "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", + "ai.telemetry.metadata.sdkVersion": "6.0.1", + "ai.telemetry.metadata.testRunId": "e2e-", + "ai.usage.cachedInputTokens": 0, + "ai.usage.inputTokens": 22, + "ai.usage.outputTokens": 7, + "ai.usage.reasoningTokens": 0, + "ai.usage.totalTokens": 29, + "gen_ai.request.max_tokens": 32, + "gen_ai.request.model": "gpt-4o-mini-2024-07-18", + "gen_ai.request.temperature": 0, + "gen_ai.response.finish_reasons": [ + "stop" + ], + "gen_ai.response.id": "resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.system": "openai.responses", + "gen_ai.usage.input_tokens": 22, + "gen_ai.usage.output_tokens": 7, + "operation.name": "ai.streamText.doStream otel-stream", + "resource.name": "otel-stream" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_5CiaobUoE1gtYzO4Ht6m2I1m", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_FSIfBdMkel3itYpiSpXVqiIR", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_G299pfseq2k27iSyvdDwKqd4", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + }, + { + "name": "ai.toolCall", + "children": [], + "attributes": { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_MATzU9ZSWZX95csufkA3UILm", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } + } + ] +} diff --git a/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.txt b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.txt new file mode 100644 index 000000000..2cce64d13 --- /dev/null +++ b/e2e/scenarios/ai-sdk-otel-export/__snapshots__/ai-sdk-v6.span-tree.txt @@ -0,0 +1,350 @@ +span_tree: +├── ai.generateText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText", +│ "ai.prompt": "{\"prompt\":\"Reply with the single token PARIS and no punctuation.\"}", +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "stop", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "PARIS", +│ "ai.settings.maxOutputTokens": 24, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-generate", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 3, +│ "ai.usage.promptTokens": 18, +│ "operation.name": "ai.generateText otel-generate", +│ "resource.name": "otel-generate" +│ } +├── ai.generateText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText", +│ "ai.prompt": "{\"system\":\"You must inspect live weather via the provided get_weather tool before responding.\",\"prompt\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}", +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d\",\"serviceTier\":\"default\"}}", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_MATzU9ZSWZX95csufkA3UILm\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 211, +│ "operation.name": "ai.generateText otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056417788193a965dc0fa61cc243\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056417788193a965dc0fa61cc243\"}}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_MATzU9ZSWZX95csufkA3UILm\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 211, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_04450c837917b018006a07056498548193bdc0e1899f73cd8d", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 211, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a07056315d4819386b39c2713805db9\"}}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_5CiaobUoE1gtYzO4Ht6m2I1m\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 172, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_04450c837917b018006a070563b6c08193ac9228622dd03bc0", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 172, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool-call\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":{\"location\":\"Paris, France\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]},{\"role\":\"tool\",\"content\":[{\"type\":\"tool-result\",\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"output\":{\"type\":\"text\",\"value\":\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"},\"providerOptions\":{\"openai\":{\"itemId\":\"fc_04450c837917b018006a070561d448819384eadfe46083410b\"}}}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_04450c837917b018006a0705626088819390136ae45bd37fb6", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a0705626088819390136ae45bd37fb6\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_G299pfseq2k27iSyvdDwKqd4\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 133, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_04450c837917b018006a0705626088819390136ae45bd37fb6", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 133, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"system\",\"content\":\"You must inspect live weather via the provided get_weather tool before responding.\"},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool for Paris, France. If you do not call the tool, the answer is invalid.\"}]}]", +│ "ai.prompt.toolChoice": "{\"type\":\"required\"}", +│ "ai.prompt.tools": [ +│ "{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get the weather for a location\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"location\":{\"type\":\"string\",\"description\":\"The city and country\"}},\"required\":[\"location\"],\"additionalProperties\":false,\"$schema\":\"http://json-schema.org/draft-07/schema#\"}}" +│ ], +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "tool-calls", +│ "ai.response.id": "resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49\",\"serviceTier\":\"default\"}}", +│ "ai.response.timestamp": "", +│ "ai.response.toolCalls": "[{\"toolCallId\":\"call_FSIfBdMkel3itYpiSpXVqiIR\",\"toolName\":\"get_weather\",\"input\":\"{\\\"location\\\":\\\"Paris, France\\\"}\"}]", +│ "ai.settings.maxOutputTokens": 128, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 8, +│ "ai.usage.promptTokens": 94, +│ "gen_ai.request.max_tokens": 128, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "tool-calls" +│ ], +│ "gen_ai.response.id": "resp_04450c837917b018006a0705616bf88193a8bb6a3f0ed09a49", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 94, +│ "gen_ai.usage.output_tokens": 8, +│ "operation.name": "ai.generateText.doGenerate otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.generateText.doGenerate +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.generateText.doGenerate", +│ "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Reply with the single token PARIS and no punctuation.\"}]}]", +│ "ai.request.headers.user-agent": "ai/6.0.1", +│ "ai.response.finishReason": "stop", +│ "ai.response.id": "resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "PARIS", +│ "ai.response.timestamp": "", +│ "ai.settings.maxOutputTokens": 24, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-generate", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.completionTokens": 3, +│ "ai.usage.promptTokens": 18, +│ "gen_ai.request.max_tokens": 24, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "stop" +│ ], +│ "gen_ai.response.id": "resp_030a53b67c188fe8006a07055f620c8194804bc4cafe07a091", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 18, +│ "gen_ai.usage.output_tokens": 3, +│ "operation.name": "ai.generateText.doGenerate otel-generate", +│ "resource.name": "otel-generate" +│ } +├── ai.streamText +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.streamText", +│ "ai.prompt": "{\"prompt\":\"Count from 1 to 3 and include the words one two three.\"}", +│ "ai.response.finishReason": "stop", +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "One, two, three.", +│ "ai.settings.maxOutputTokens": 32, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-stream", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.cachedInputTokens": 0, +│ "ai.usage.inputTokens": 22, +│ "ai.usage.outputTokens": 7, +│ "ai.usage.reasoningTokens": 0, +│ "ai.usage.totalTokens": 29, +│ "operation.name": "ai.streamText otel-stream", +│ "resource.name": "otel-stream" +│ } +├── ai.streamText.doStream +│ attributes: { +│ "ai.model.id": "gpt-4o-mini-2024-07-18", +│ "ai.model.provider": "openai.responses", +│ "ai.operationId": "ai.streamText.doStream", +│ "ai.prompt.messages": "[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Count from 1 to 3 and include the words one two three.\"}]}]", +│ "ai.response.avgOutputTokensPerSecond": 0, +│ "ai.response.finishReason": "stop", +│ "ai.response.id": "resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60", +│ "ai.response.model": "gpt-4o-mini-2024-07-18", +│ "ai.response.msToFinish": 0, +│ "ai.response.msToFirstChunk": 0, +│ "ai.response.providerMetadata": "{\"openai\":{\"responseId\":\"resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60\",\"serviceTier\":\"default\"}}", +│ "ai.response.text": "One, two, three.", +│ "ai.response.timestamp": "", +│ "ai.settings.maxOutputTokens": 32, +│ "ai.settings.maxRetries": 2, +│ "ai.settings.temperature": 0, +│ "ai.telemetry.functionId": "otel-stream", +│ "ai.telemetry.metadata.scenario": "ai-sdk-otel-export", +│ "ai.telemetry.metadata.sdkVersion": "6.0.1", +│ "ai.telemetry.metadata.testRunId": "e2e-", +│ "ai.usage.cachedInputTokens": 0, +│ "ai.usage.inputTokens": 22, +│ "ai.usage.outputTokens": 7, +│ "ai.usage.reasoningTokens": 0, +│ "ai.usage.totalTokens": 29, +│ "gen_ai.request.max_tokens": 32, +│ "gen_ai.request.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.request.temperature": 0, +│ "gen_ai.response.finish_reasons": [ +│ "stop" +│ ], +│ "gen_ai.response.id": "resp_0c9f92fb68b418f1006a07056074008193a2191ff16a93fc60", +│ "gen_ai.response.model": "gpt-4o-mini-2024-07-18", +│ "gen_ai.system": "openai.responses", +│ "gen_ai.usage.input_tokens": 22, +│ "gen_ai.usage.output_tokens": 7, +│ "operation.name": "ai.streamText.doStream otel-stream", +│ "resource.name": "otel-stream" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_5CiaobUoE1gtYzO4Ht6m2I1m", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_FSIfBdMkel3itYpiSpXVqiIR", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +├── ai.toolCall +│ attributes: { +│ "ai.operationId": "ai.toolCall", +│ "ai.telemetry.functionId": "otel-tool", +│ "ai.toolCall.args": "{\"location\":\"Paris, France\"}", +│ "ai.toolCall.id": "call_G299pfseq2k27iSyvdDwKqd4", +│ "ai.toolCall.name": "get_weather", +│ "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", +│ "operation.name": "ai.toolCall otel-tool", +│ "resource.name": "otel-tool" +│ } +└── ai.toolCall + attributes: { + "ai.operationId": "ai.toolCall", + "ai.telemetry.functionId": "otel-tool", + "ai.toolCall.args": "{\"location\":\"Paris, France\"}", + "ai.toolCall.id": "call_MATzU9ZSWZX95csufkA3UILm", + "ai.toolCall.name": "get_weather", + "ai.toolCall.result": "\"{\\\"condition\\\":\\\"sunny\\\",\\\"location\\\":\\\"Paris, France\\\",\\\"temperatureC\\\":22}\"", + "operation.name": "ai.toolCall otel-tool", + "resource.name": "otel-tool" + } diff --git a/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts b/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts index 93d84a826..2634d344a 100644 --- a/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts +++ b/e2e/scenarios/ai-sdk-otel-export/scenario.test.ts @@ -4,12 +4,17 @@ import { matchFileSnapshot, resolveFileSnapshotPath, } from "../../helpers/file-snapshot"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { prepareScenarioDir, readInstalledPackageVersion, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { extractOtelSpans, summarizeRequest, @@ -124,24 +129,65 @@ for (const scenario of scenarios) { ); expect(nonAIRootSpans).toHaveLength(0); - // Snapshot the span names and key structure (not full payloads, since - // response content and token counts are non-deterministic). - // Sort for determinism — OTel spans arrive in non-deterministic order. - const spanSummary = allSpans - .map((span) => ({ - hasParent: !!span.parentSpanId, + const spanTreeEntries = allSpans.map((span, index): SpanTreeEntry => { + const spanId = span.spanId ?? `otel-span-${index}`; + const attributes = Object.fromEntries( + Object.entries(span.attributes) + .map(([key, value]) => [ + key, + key === "ai.response.avgOutputTokensPerSecond" || + key === "ai.response.msToFinish" || + key === "ai.response.msToFirstChunk" + ? 0 + : value, + ]) + .sort(([left], [right]) => left.localeCompare(right)), + ); + const event: CapturedLogEvent = { + apiVersion: 0, + isMerge: false, + row: { + span_attributes: { + name: span.name, + }, + span_id: spanId, + span_parents: span.parentSpanId ? [span.parentSpanId] : [], + root_span_id: span.traceId, + }, + span: { + ended: true, + id: spanId, + name: span.name, + parentIds: span.parentSpanId ? [span.parentSpanId] : [], + rootId: span.traceId, + started: true, + }, + }; + + return { + event, + fields: { + span_attributes: attributes, + }, name: span.name, - })) - .sort((a, b) => - a.name !== b.name - ? a.name.localeCompare(b.name) - : Number(a.hasParent) - Number(b.hasParent), + }; + }); + spanTreeEntries.sort((left, right) => { + const leftAttributes = JSON.stringify(left.fields?.span_attributes); + const rightAttributes = JSON.stringify( + right.fields?.span_attributes, ); - await matchFileSnapshot( - formatJsonFileSnapshot(spanSummary), + return ( + (left.name ?? "").localeCompare(right.name ?? "") || + leftAttributes.localeCompare(rightAttributes) + ); + }); + + await matchSpanTreeSnapshot( + spanTreeEntries, resolveFileSnapshotPath( import.meta.url, - `${scenario.dependencyName}.otel-spans.json`, + `${scenario.dependencyName}.span-tree.json`, ), ); diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json deleted file mode 100644 index bd580cbc8..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json +++ /dev/null @@ -1,299 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json deleted file mode 100644 index 50ae8de47..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json +++ /dev/null @@ -1,243 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json new file mode 100644 index 000000000..0c13cfb2b --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.json @@ -0,0 +1,389 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 27, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1416 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt new file mode 100644 index 000000000..7a2824c37 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-tree.txt @@ -0,0 +1,325 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 27, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1416 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + └── anthropic-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ] + output: { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + } + metadata: { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + } + metrics: { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json deleted file mode 100644 index 748d47113..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json +++ /dev/null @@ -1,383 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 5, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "start": 0, - "time_to_first_token": 0, - "tokens": 18 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "BETA", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json deleted file mode 100644 index 10d51bfdc..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json +++ /dev/null @@ -1,319 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json new file mode 100644 index 000000000..15d3e0f66 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.json @@ -0,0 +1,473 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 27, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1416 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "BETA", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 5, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 + } + } + ], + "metadata": { + "operation": "beta-create", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "beta-stream", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt new file mode 100644 index 000000000..fcee56e52 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-tree.txt @@ -0,0 +1,393 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 27, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1416 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + ├── anthropic-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 589, + │ "time_to_first_token": 0, + │ "tokens": 645 + │ } + ├── anthropic-beta-create-operation + │ metadata: { + │ "operation": "beta-create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BETA.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "BETA", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 + │ } + └── anthropic-beta-stream-operation + metadata: { + "operation": "beta-stream", + "testRunId": "" + } + └── anthropic.messages.create [llm] + input: [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ] + output: "1 one\n2 two\n3 three" + metadata: { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + } + metrics: { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json deleted file mode 100644 index 7b7fd46d3..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json +++ /dev/null @@ -1,605 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-thinking" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-thinking-operation", - "type": null - }, - { - "input": [ - { - "content": "What is 2+2? Reply with the number only.", - "role": "user" - } - ], - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 0, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 49, - "start": 0, - "time_to_first_token": 0, - "tokens": 0 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "thinking": "", - "type": "thinking" - }, - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 5, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "start": 0, - "time_to_first_token": 0, - "tokens": 18 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "BETA", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "beta-tool-runner" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-tool-runner-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 72, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1294, - "start": 0, - "time_to_first_token": 0, - "tokens": 1366 - }, - "name": "anthropic.beta.messages.toolRunner", - "output": { - "content": [ - { - "text": "The weather in Paris, France is 18C and sunny.", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "provider": "anthropic" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": "The weather in Paris, France is 18C and sunny.", - "type": "tool" - }, - { - "metadata": null, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "get_weather.lookup", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 607, - "start": 0, - "time_to_first_token": 0, - "tokens": 663 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - }, - { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "The weather in Paris, France is 18C and sunny.", - "tool_use_id": "", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 16, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 703 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json deleted file mode 100644 index 0acff38b6..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json +++ /dev/null @@ -1,471 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-thinking" - }, - "metric_keys": [], - "name": "anthropic-stream-thinking-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-tool-runner" - }, - "metric_keys": [], - "name": "anthropic-beta-tool-runner-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.toolRunner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "provider": "anthropic" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "get_weather.lookup", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json new file mode 100644 index 000000000..654a83d5c --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.json @@ -0,0 +1,896 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1415 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "anthropic-server-tool-use-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "query": "AI SDK release 2026" + }, + "name": "web_search", + "type": "server_tool_use" + }, + { + "caller": { + "type": "direct" + }, + "content": [ + { + "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPeyh/w0GB57bzLO8xoMzb5iJ0f/kYhmTxsJIjCY/9gvu/eX3XWhg2YiBX86CXLEMor8pKwRIhRKfWQYhB1H1ysVE+Kdn84gvzbUY9cqzgjJNuI3vWYVhf5p6prt6WwG4hj0zVOTDx63JOlg3vDdvBnhlsH/7K3RHdQakLTP+jlLwROuVHWj6NHrow+9NxwVl5lFFUUFWsv+f+oSd08QI9CrxsCdX1zOdOHU4xgGtWiUaxPV0L+chRclHYIXxKTAEKCBFUZCdJNFq3J+Gh/qEy09DwTkMlYLnf1lnnchVXzSFcDloB3aO4NCOQEnkf39gSjgyT7KF+QlOitxPBKZV0f0vlglz5Qp3WgrqCGq5y2dzN1HnAc/vzTa2z9iy+sBCC47QlHLpI94eGBJlYRUcsnyEgHgJ2/jcudAUEfNtcgDoNuEP9hFsvaQs9Xa8DEzl6pR2g+HpzuRzuFAGFodjzudFFWoN6aG/HMvp+wuttY25heSNOgeZ8/c0Mcy4hG0aBLj9zZ/9dTbef99Ewfx/TGxZd+sAMihGXw2ZXHYlc9m2GhjCaQMkbBJfAVasMHmGpF0q+knR9sU7t8Nva2KTEF+IuBYaJikuj/xxL/a7OTMOR8zvn/JSVgRw6SgTviJycM4X+iNFVy2nrp+YpdxJAjWVXCJUsd6YUDKsIgUKVQfqkUNBFL0LhcE9eWNFmptyZIWC/pjvi73NVNwb2N06VGD5oa9Iac2PQ8XCnIENLv9IE3uFLR9bOYl0LR8iai81VPO3l5QHJJcdrS+vARTGrW4Ls8LYOy+ZLAHz7eFW2Qy63siqK9AivmMO3fJlMcFtYSIiATSdSJxqHXjimdi+qs9v0Nz66rmj2uX9Wisp1fH05YY4TcJ1GUzwojKZcaI/X8pfyIl5921qPZl6C/8BHt6nUWzdwqDS8WhqRr8x2ayFO4SDkMhjXaG/+8+ut7ETbJcMEfme32jmyT9tyuQrOp6amwwyoWNbQIkoDpIX394x+lo26u1wtMkjTRFOScs1bFaDXM8WQphUj4qBeMWhNaAwfr1klpArT327vQKZxr7FxfOuaMeswGPwObGGz34u6Vat+MwjIZhIm793O1HzqlLIRgoJkUyVA3d3/03vSjen+/Mxrjeu6mn+vXwH6SfF9Sh47WoLQbPRHQ5hOJQHoPbXsAcSGGi+q9HnChmcneC3Qq70PlaFA7/iavS+t274TE0FLf2BLnGuQyu6jYW4KHxIlx6N3m9nYMuwTgPI32DUoxFdUlvEMi2wARnJxjai1l0sofmSUp1gpIHZqw2coMsY0MfdONJxO34yIIT8GQaANZfemmcsFflOagPKnBQdY7uwsR0WDyH/asgn80SjTUCVUpgwZi2EmYqBPzq67UinGuePSMel2MgTfmPMkyN5ilZNsrou5qGVlv7wVSDsL5eZgtbZIJJpWKe1H1X51R155n/gn5OGJaE2o8TGsxM9IqUBs2Wtw1qhy6eBUxxoADrIczTiLkUg9UibRFKfqvHGJgF/uQQ4sKo8YZ+aprskETHLxgINxTrQ4GzB/SpFtF7TfcnhQvQO4+H/tVcGAM=", + "page_age": null, + "title": "Releases · vercel/ai", + "type": "web_search_result", + "url": "https://github.com/vercel/ai/releases" + }, + { + "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCFspfWt+JmUmC2I3xoMvPoLUtU0YD0FTFdOIjAr70jgxNqOtajClAfQMqiGoNM9zVbidTf044FsdHPkd4bPd6JjIsIytDLC5N9pQkQqkRnINNHuDrVsjmRXcm09BiCqyb4bUgf3uYUcf8wOxwNhOGPhswN0r5w9ghVXBy+22wWmUSKbw4OPJ/nf2DtPCv9/Qe/VSRKf1W/y2I3oifu8ULJd5ijLecm19kXW7zYeMgdye477SxiNCmZudz6XJh8GCCKOjDlEGjY2dCYRFfaasBl8lGLi3kNrCQxUy2oPGmUIJdPZrO+u7AlUE9ePyikJhUzGxUTOd2wxF88rIBcex9vDkQqMT4m8Pw5u06pH3ayuwY9qUm9yH/euvETlZdK6w2lSfFHIKn5aEnMqvzq7sIH0NQAd5McNiBedbDk5vpc/eD1093YG1Hne1bAJt3sccMakUC8wGkPZ3F3PSWBT7CRmyaWTuoEVRJRPB3xBvq1cJiWa8mCWfHSP3UHnmkKePRIt/YsZx1aFOed5Cz3Mo0xDpSAEdahL5ot2XDyKN0wpm2NpkKMsz+9fmg7RvJNYAtlZjDuaL6TmxUYgmFmyjYh62FrMijEzQA6AJesCNmAjpIiYKEwoHV6QKzRxPC3TYbityHuE2j0JHzHqk9q1nq1aKQiRyxwW0KmrjNF1ZTi7BfklEL1NUyM0el/PiQUPBGP2TEO/1NqymzxB60Nvj0GLDmIblsW8FWIHEvi7A6W6/n6F7T4E1tOZwCgqSqR02IAPmK4z0eyH0j0e/ejdzS+QxtNnXbopw/mFdiZh2W5L5GoeS6DvZp2HAstnxnq+5ye7/5sLso1cSCSv7rUJoPo5/U/StEiRxhOJSRFn8p6jGl+ZD41EhuhRy0CMEsxG4GUTnf7ZbGzeJvi1xndxaOL3bc25Ga59VlMe+Gdni38uoPH0kPy69KRJkFUl6JiwRHs0jiQvTvVHND/pLMoLOJY8GkmZFduB2PkzOwjNyX4Q8rRFJ6vgHtNvY4RURdgo43un74FfkNldGNOxcW6MVmsC97tDJU9Mg8TNYI5VB0iYEmxQ8rAIkJ5NLRs6B6JiHffFqiJEGXimw8VL8Tjav+SPVXKx2pZAF1+9ByG8+7ybG8Ekom0f9HH+uxgEVBKcClMWeC3V+ZL/U7GrwLffZbFVvYOTsbBF4WXliGIfLwsrSx49vXgXOQNtUlbHkF+yNN2B8IkiN78FxOPQeYidhh0QYqEyzol/NpA90F4EgbTlw3CDtcLNrYQaWTadEFPCroseKybQbeX+RLvtgN2u/gegLbqF6Ktwt6Gq0dPdL0o1UdoF6cH6CSymn39ZI2X4YDuNn1++Q/VROc0yEdMjG29cY8agoJgEV/64ctgUIPsJUF1dkhsL51At9iwrq7NJMnL/nVxkqg/UMbWRuAGAitnVxx26AKgNiP/Qw28DqYh/e47z65HNgKDZiJVY88N6Nvb3NqMFaPUbhS/Ch7G2W8q2hy84G9pxU1Do8J9DpPYr+HdX/RaBBTokJd/T1sCTYQm44n0z0KosXm7kv3ZeBYJYjFumrfQCkY/oW3tHtnOMEbrXH8tuPVSezYetEVScLJk+n4yvXCzS0wSQL0/Ei1H7aDzmN3f4ZbtC+vD4UlzAD3P7rLgBY5y5SCxWjNifsiAJJgWsHyXYdOfJQYjKnqVvpO3nGNO57VGKUTkNUdJGzD+QhqZeGuZsc8CDJSEFjpA1D6ENjhttq64UxoOUZF2CKLGv9jfMIms/NmAmv7KtsafuYd1eW3qoC4U1My/nbjpZBrqORlSIuWnDe/3IqEEc3lP8gK/9W/58rEiGsMBOh8UO/8bJCx6+UrlZxV1SWddPxmVqpVt0MbyrrglbYrp1jI/74GsSKrBhjbUYUkJPvLClne2OnTtJisuNG+5QpTH1aOh4d5CsAZrHtMJ9YBSw9dT1Sy6ZdiHm1J53+qG/wApdAnD0Vuyzbv5w0tlBlCB9xtOTeU4b8tKpmaeMiI+m52sYNJLAh6OENKpOxdbKpvhB8RiAnPdoeFamoUVa40YzcYvpioWu8je2irv91dJcH27CmJadDBxARfwxoEDsfJq0AL1d/pUfBx/27jVBVolCZ0mDaTWLKiMp+Xx0GW7SoQ+WtjyKB89xgO4qyD2A7t9K5H/5HhbpeTA29oyTnnkQyWHdVtBreUCOwwe6M4/XwF1Vwby6CnWQCLZZuBAT2XK8pIPS4g25oIHgNlEeo6doH5vauuodXyPR6hMebgf8PNskzrqNwyevkdaQ/zTGXy161a2Hj9fpWzSFW7vZ0y2WEkiCFFBdihQaXXgr/fiTGsEdPhmrwR13gL4sazVKI/WYONf6nxpDQPhSAjbGT72CWBY39OR07VXPeErf44BhL5OalmNMEbSLWm8r0r3DthXxoVvyur0Nxu5vQKBvQVAR1C+UuR/JagNZcT2heYNLZJVxp0CGMf6jVe6GOZis35qdaZFpkvAnMAWsAtwfiaZpahzU/h1X9tFZVihlzhHJoKYAjmwrLEMJTTMZptjB3+gA56+FA1r0hTIzoxsG1u48BzB5sT/aeg8GdCUHZM8ozMgc94LOu+1WI/iCin7KsNcA3BkdRwB1tZ9DfcdD5wJDha2E4yzETPeCgyEcGrcogfw71hak3cB3a33iyxIMWdD/BeLDxvWKA6uCYJdUsyyUIbHJ9fizErPxtZI0NxF0nt0Otq2Wkd1KpAOaSH58cnDnQM9BlkMHpddr5KDTSh/r0JNVentvMdnmzEThXYKzDDcFo2XZMa0R9S/c1jJDJrqVx8rzk4yaIV5ydOSWerbtm6ZJz0KUvsz3q5A95UC6i7pcGXaGbAZm48QJou8CS5UAGtTDqC2Qp7g6IVt+gZkla2F+rNZCd7gbBBkSHF7LaYQ0ZV5qhLkZ7oODArziUgIFVKdVn/bOuYKV21RjLuoS0nPKHkBB8gVl+8robA8efKv7u11NIzgD4OyWqhJZBO/tfBBtKvA5I9ehubsMBhmIyoFqHfOfpUDU5dvlmNFJqjDq8u7WUCUmf1OJIl7L7FiUL2T0+M8qOYVSaWhYa4xy04VmwY5q9CIKDvYks0pMqj6VCOoQzZpOIUJ8VyymCTQ10PtGj0V8tIsS1rTRo1swjThpUTx46MfQYJB0Wa5LWvp8jjn+pDQ3VODvolYtCoPty1Lll/oP1Gl/gJnWKzhPEsL44/IunxJI9k4I3kwzXCpUS26/PRNYEXXWMSkMNIbgkKDqa2Xp1UeSHoXNaUgDQB70uc3yvNhfKVqlUGvSb2NPNj4m0wfzLLiA8CFnIfLyVFndo6zzR4CXCIrGBJgXd3IbmIJ4kXr+QLxeQizqSsmTeI3c1lcYbRPm1btafQf8ra3sQpXD9Xau7GAICBxYfS8RyySYhpdDyNsWLyh5JQivD9iYRQA85rWiUNIf3R4dFnPrVY+I+qThjpbOFLQQRqrlI5SnoEWpnvaIFydPlHHwsHT1Jif5lVfG7rCAV9w8XrnjKhvtByZPwwp+QTzDABxwLfA6tpEqI5bFmtxsJyZJ6Dhug4yTw6VXIXjeTyyuGym9cCqWVrUyqvFnnzEPG9POLl5Jn0es90s3hLHf6JM0yRrdY5uMV/8pNaxcl5W/UIRmJLH8DLr9KtsR7EF19NHaBy5s1fWqNnJ2TaQ3FjvyB8v9PlPHxhPFeezjme+MfPcuB3egzhsgTo8/J5b0NzjbX/vkTX4riCURKbLT4+V/kAPdBH6Wi4C41M+fBSY6D1WtgwCc3sfL68Xxri8Lxq5SqoJXt9hwetr7kLmgcDRIi/vHVxQCfdhsQdw3IOxIAsPygtiVU7GT8WgjRnWA3/G3EWneNJ8a+5KfwN7zOgqn2pO+BvLQz8o5ZJ6Ve32XuU5FHf+BkRrTpXlpfnIgXfZ9qfWGHSj/Fmtw+/e8x3rRfe6+pkgrQd7rwXm2wbLb9Y7PwunftE219tuIMT92ABbJEcPw+yueQyNXYYhkW8icZbnfR3V5u4+Nl0aBOHO2rFbuhoa4+I2XncoFYXEYPdNm8lcAoA5AV/Gc6dV8kijN3gk+Vmk2TbLAP8AXyP/nxLKylVtyltM/p6wd4nxDjJRrCmQBBR/sq7gBoKpo+saSf4Xtv/HKNijUAjNYPbp98buOTCFOhJ9VIYcLDueBlPJTuxOkuLLpl6gNsOs4jNa5xGRectFUUbExHxyqWm/N0rbEIorGNFHz2LywD5KJFX/cYsQepy5POBzJ0i1H2NxDIb/HNZK1s4pnUgLGjo6r7tR/cy0O9H6MvfjTOqqslu5bI+yICnH8RUvV8rA90U7J4saXY+Md79uoYAEsJl9mSmk1HorSb8zrpOv/JaEjAJrapPDC3QcIQF7LKAuk3GM/a98Yu2jRZDBWsO7RGAM=", + "page_age": "2 days ago", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + }, + { + "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGqeizt50On98+WNdBoMrsFK9FVnQtBVEAurIjDFLw0WQmyfselv9Y5iYvLTMXJ5/E97vJi+28bY+3LiKoqdTj3dxo3DHvLH/01NRgMq0x46TJJ7syYvxhP3vu3rol9TFY5AF5RFxDssQrgdmSto8+X67nvWXlak86chMf+RDSriC0v/OLoGOoFfvcbRfi7iazvDmZSXG6XfVHoieREerkcF/d4yeKKdphiTWnjOld71JqIktcTOTC/SyQgtfnUyk81X2VLGQBX7lEXO8+urboSL+muUvuHedmRn4Jqk32C8WktvIqfJXdUgerRErO8dPQFqdCaWabjitcM10dJ63KWBBPFq6FdjZvnuBya79+x53YxeSJUrr7x/waRQQprcTT00rU+r4JwR+qDoUWi8Xvj+Sxo5AeBeNgJEOgdQJe/gwT74EdcGqd3ya3d8bHghBHAttrv5s3Tr3HjGfP5m7rMSomKbjHBh4IlSWcP88WRRDGnv/eOMBhzCViQ3+J5ofPsngcuhiAxFRCoGavRE7MAjf9XFBonxZbUAvSGj+G+9/nOOi+3TiVWA/OjNhhPFNTNQHW48QdrnR8Ty5K2Bt+KZny9EYbAnQDkBXgju3SIq7TnznUggl50IifdJqP6dGjzih6wJKoU6Qx6b7X3XR09hrETTGTbuVTMJtzAdcqib79YJUsRItrkgdeKTmwe6g1i7u8J+gyxAIYRFG3VbRqN/SQkQAcgu4c6rcp9+hiwkecSDjau+uuEoGA/w/hf7rkJO0KfkkIPuExI+wQjnLo+avekXfZpNQJpsa4VsSg5rwcMswRRRbnQGkpmShHXxYDBoQBV0NrMWCpdxvTZO9pyrJzhRLYlySQKQuoDCIAuSjCuou9SAuv2Vy0pTxIcC0toIg0Dd/5rrILxd0eu8Wze4L06SniWOUOlH8PLLmLqRoLPpMiatLBZX9H0OB//hHi0EbMHUq/L5TgVa52iR7/T2GKyptwEM1Ivavl5YOau83WBwy0BvpALRKuM6jEn88II6ayiA/GBg/ImuW/0aTZ19DBPCLkkm4RlzKyLUkpUzuBpfekvxPq/yG1zviK85pKtVax4lQqK1BIKudSOXvBnyfQRKubFprDDvxln89NSrqUP0ZMYbeCxFxxLOinPRZyRVOZoCoG1dFFANTy29DKI5L6KmvSd/KoWP+6jgyU2V/gpIQEvWodkgZUjWDBdxn3KAOPTOXPL2jZRLJb8LKlOeOr3KyDKa/nPsAVvKtLjOWKPPiF4/vL3BP3nJvphyLUTrOU8Bcvfl3z/qPZx4TtlRPW3AS2avnLuVM2Jq7aSThdx16vr47nc/zAgNHq/mFvCLLgQBS0SktpUKXPlL/E83qDOZEsNIEi4IvYYxBb+ml5ynhPwpMq48b3kPfcVB0/nIDvTJo//wZn5J7sKhBRrSAiORo5XggJp+VElaUDcunET2YgkHJpMZZlG7CDOmiI4jCoHPfRpE3oSMeXnzOuc25Uibh3kWkyvnP3+hcNXtmREIY5ai5A7vOEABirvusZThOruNuemvRjhsiRYLWOJ7q8rcaFiOn4kNpozwFRnaVrb4S4CIJs5ei+clLuOJdmRtFIImUkntZHBpr96YxMkZBjE/iwoelmI3Is1S0xpIPT8dVmRfyrgr0GbLqn7Ws33eJwGR8EssWjJyk5NiSh/TelT/rP17VzypmGrthU19R+7adbac16/DUZbAFpKQ0Yrj+cYoxZ74eyTfC5C4so1P7yvTuxMNmSr2R+cYurysTgECgOzH8gRx/ej7aeRxn5CEQKTeO1x8OKkc7B3ELTKOzHY2UxqPGFKAeM5nuYMJ7v8r0qq+6MX8g+MpWfmCTMhihLC+PSunQYTHzRy20LpC7Y6fkMlfPCUL2awB5wx2ZOzAJegRqANDd2ARZMfT4dvUPFjZbD3sU1nBsfh84Hn+WqOb1fA2DBufX8b3iX8+T+OgXk7z7ZKxO2PGz3ZjAORQzvLrsU31Z8DdJoi+Z120m/VXdHyO2yecPJqsJJrW0pZ2HgvwuBrUjGSudJK4z+UxKxlmGc1DwLkePPWfRLbQYItsPrB6iH2GuZAn7SipmgYOHqgtWqB4J6vR0UznXwDPHRedcnMwNAk+vAOjXzdOPfjduAwZODVaP3guRHr+4rMdwFOtq7kGuY5GO10cTmCg9d4Tc2SwYmta6Qn4tcoJUC7mQw26fV89gzktn8qh+WUlQ2St7Au/vSZvqjZz9trOKyekqn6/IJGhO6ea2TqaeNk0Q/piEMIVqEmUmihwgaVWOlPDN8WACOuubYrTwx6lWvHEQfq+aSNkVZhEHv9AzgOWs4yrp8BENTyyuQWYvZpH2PvSbNpeva2gZBVDXdxJfV8Smz+8Kfs4E/56alj0WV3M/GDee26YRqOO1j2VmK86Y1vj0GvX9WqIH93qhvFdjwhDFvv06KwVypO4MaaQ93qggqiwfhrpz2jDlW9KxpE+Jo0WeuuVANvo3WYsYtoZVqrE8SOyfto3bf5zd3zzwONhwuVmp+yldoNRJfPkAJeq3Y8SQHtevPB+t/M3poouruv3JZq9/7fUFZItriwawRoqbb8SIc9IugXXmwgZEJHt0z2EENHDLCvELW2m9Tiy3Oz9naWfrvmcdpnehL/wFneKQyy83moZ6JZb9dQqEKsIkUPhpSy+25nw04QYwT+MLth/L0cEI9kU0coSvWZNkOgQWrsRnnnfDMANOeYT2WeX6aREtnAwxYXK8EEx4L5Ds7xSPDT6ytL0+bjSgyU8sY5Aankmd9vfg5lKfAmRpiHWu3lWdaVNnHaJRBRMjIJTBWBjIwXvVHuaIJjEl1XXF4DlRVUN3YQIt4w9/gxhmX7nIbiVFCLT9QRAScSoY/bYkiq1VPnfJHCozdaUjKuOBMuYyX3Sn2LLXa67JSYL86//w/TvPWpffSihihKaNkNAgXfXMITTKjyaq1Ll5AdGGdGS53/sxPz0G7mUv1Egpe/r/bh1bh/M2SH3ZnlOvHQUehHPuLA+e7b+E7aTteSamUffdFxtghaoCzwHFp2Af9zmznILAgkw6cfwCvrtMmVNuH1QuXR+rwuZMGH0RXg3SQBoW0BZGQIqv4CYHLX7XJCOtTfA4MUIMNIv2jProhVER2IL7Bi79e2R0o5dmPtSxwfYewKHypGFr8j7S72WUL1S7zkvmnTDr5kHdBXsOCNXQk+Bjg/TAOzEgS13rFXbiDwAcfzRbH9fNNDQ23EVr3DyS/qoZHNKkQrbwJPF9dErwhs3hMMiHTSOkJOcWNlDCjil2e19SA7jts42UvrxJlq06iIr1wMUbM+hcGxN1JHPjuyFS/KjcuNyck6reUi8rraXaoe0Jt2pv2UXm2hX33J3Zc4cqOzEjZQu4cxFQbUAfDcGiSBJiJv8rWxtrvVRA7RleEAj7fs5CIlUf92dQjVnX8cB/eYzfj3tqPjAqdhFFX87kAuAGYOIBR2c3HtMkv6FKe97EXTOOyDDKxDVW/TL/xqRU+1++KUfLbobT0GM0OvOZ8o9gYleQy8dJ/WiT+chGCjKGWOTaoZodIzlb8OMUUF2EawzDmp2N6qcnToV7lFdN9cUz3RJt3gaIDYqGrdA2X7hXBpgura3OiQ6Za2sTAzqZ1o7qy/KABTJEACghkKXDRDjw8WmJnJQViaFM4+u6wfQYETddumrNnxNXPQ8jx3v+gt4pmvALcFBlmsPUJhx8PbULjdgXQ8q7hCgkp2UTM4wQnPBnobTtHMzOejyO1M+lS4WSTEETRqcLtYLt+OXYfXZpdG0OsRPUCHJweJ3dS9V6a0PpuClJAfH4+S8Cr7guDznVVryfg22e3BHuAhQOZ3VIidxeBTEP73EP7x4Hih/2A17i5bpzbrB71WvaQZ8DpX8yYoQ1NkTibzZJhzgUMGreKY4gdCfPLlfj/+in6qv3UnCCpJwuRXERleuUoZE0BY9U3jFO0dmRiYTiLk8ypGpIJ/frLJHV3QnxmftfomZvO/fMRYGWKMDZOGy638lgA7od3ZnbKiuzpJfZbcUN1D6OR3Gueyqdxi0uKkeZMrtEEzFbxdHjncAmKz9gV7a7L6Ln1Lk2w94DPyh11naHqB6K22eqyYtSwJBPfmwxdA79KYLxEooy/8Wk5JK25PDHECVQqJ3CLCpPLdoRv4YMYH/R8GWyDSYymC4zs7V/dMM7bcwelA04FU+tc266jQ6z9QmCJfKQYk4mw07z9rzsV2q+AXiNiwzgYS5zDmnd+ggjk8c1UBK1MfDXs6Mgunz8uObVEcfWKgrfQbn0S7NDnd+SCCySkQDNvMy79D0BrmdUNC6uMMMOq/dJTuZhKI6d9ZJAwDyqrgtxb4o0ZO2k4H8yderS4bxcs3Z7INhlU7DO9boWl1RwCD3a0oo+rYCMPKFpyfHDfFwWJfkFs2n4Ir26ZNRjwjVt5+ZM6QNBvq0L7HN7wxAt19Jpvezl6dMef08VzGNcru5wnJYnG1NyHY2KJ6q2gOfrk6Ca9GiJfRfwmsvZA6GxRXw8LmlA6oNLkhUMcVJ5KeMiMZDxcmoAbBytfQzFApzr/8o5oUoH0EeQNeq1O19Q3Q5RWLUepS5uIw4Zd0+NBFTKwVPWe261soYFP9zuC8NrX0ABD7K1MgIYNHXC+YbMhGAPF2IvBBcVCbKfiIdnypxfzp8bUx5n3O4OPFHGppmHCtbgIGTxm1El1rrSklwUX2byezD6vlSH/218YJt84WblycEwLTznMVmIRRI7xS2pY/Zy/V1tZDPGbQHtke/v6PT0OcAPfGESnXAiz/g9722qx4Bn8CqxThGL3V/1dAPLOJbMgLrfGIHU+GX3ysIW+V+kBIO/n4U/0OdFK0/Ux4S5de96SPRRdTSHriRh6iw8IHiyIKbhmM+0mxUrnPEmsdq81hAb9Vxh4aS1b/OGIhMbx+0SDqNshTPuftQ9k22Xt+foESADbYmNw7STJ2gt549U30XMobhrSSso1Us08WAU+suayA1hkMOvmsXfOQoJpvyXDfxtD2d1IiZFtjAVkn9QFMC720caguWLQZmd5/fhm/IkJh0ixB9QsLvvECJueJcBHUSfu7cX5DXcoR9d6O0SW4HBnGgkPPb5KfwtyZqB3A7nb8BD35YAX4sIOCBBdxDfNfRe/+JXfU346oXw6onR75Qvn5XBBVe8jbPqbkjmkd846u3Rnqa36+uJTUB2/XvdH7ByLOlp7JyuN4VbvPFMwFDQ+xSnKqYjFsCvR3SZlDktqpFZjMXUpmgG3Swxzj+UGWQN1HkWWBwP0bvMsWrDbThRmxzJDY7wSf+XSwLKQc9/xlPvLGuVGstUECy8ABbDhgD", + "page_age": null, + "title": "Release Notes | SAP Cloud SDK for AI", + "type": "web_search_result", + "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + }, + { + "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDN9ol0uSpJZ2+v5t8BoMq2HRp7ONJ8GgQ6+bIjAS2xnGjGTBkt+t+Xr+AhxgLDp/9pwvW53UFx7rNu0kgJdhrpG0KLbi6MyDCSA1QuoqyhT/5oaBOjOAbQfNT8ZrHI4j6lzvAHfrm/HyPJzXYuY/ZTHryyO1JjP2tuM2t9C8/fKnLLVko8fx4wdAwVgh+2YqPWJBU8lrUXRtRhB5QMCCtT6OSmKZSYPs+9LOiIoheIV4vo4vtiYhDNcNlZNrYMYAB/KHgGdeWq7w8qdhziiPSBR67wsn6HbPajJ45OD1Aqd5ly1VCet5eh4uZPpXu2dwapPHUs2xRTGd6GRnkpe9VXjRtbKEmKhab4iP0FMU940MpVFTV5adJtUO0wsce1+YxTmHiRwf4qbWhanQzCm/cd/1Upk5eCu5K2EH+xY53HC0ADFyxfkJVMBowvYDngLMolSNelRqsTQLvXWoNBR7qkmH1HWwAuAy1ktIkARPcVgZXixs4elkmrgipJZxaq8FwyYmjNZaFUnfRh8+0ExUNNKGk5bQPmGXppHkc/3I7BH68j/ranbUUrXxw/s5acAacu0ZxRGFOx+0DS1nylIyfiAaW1yrbPs+tn0w9hSrUEd4UiiffYLEZyjcWHXL4HQ13KjEHO54gSqUV0dy3fOs05ld9K11Sb5B6ve06fBwxgWGE7bSTm0asZNc0AzlpgA3qELRdzt3nCTRp5tzW41FnOhfRt1n+xw6dm6OZ88RSrTia8ICz9FxjsJmLl6Zcug+7qTaR2f/sK/eMNqoz97vqaj3oMCiigSYGRbQ9VTGVgBQBRmPXGzdZIvjM1B/CAFjKKTbH3175B6fdZZp6tiBM5APB5bSh6OQ28mXBoIHvyUfdVj9FHOLhp/KnmU3sAbkjvRGLQh/YrgAmfMRjBM2OrbaVyL/s0DovBcHtaw+GcrsN8keDPd5gk0qgj2B4i163NSlMPMgbuV2DhRIInc/Z0IuSm03TuvLvDkjv0X0MUbkb5vK0pAAV6zhnTTWsSp2iHxEuzpfUsC7XWTPb5HB36fyle79OXky2AL+C5amspqXyMnxgx1Yd92b7kHP49UuhAswSOYGPdITIwBfHu1b/103OWsHLo1sU5bpKGe3wJX3XuMSW4Km5cGqX1OVBqFsU5rCeBxmULAE5o2f3bxSGVJPYEoAsw1uYM0oDcfDLf/6eEFauRu8gI3X/FKn4nL611I1IqXUyB24BSGyGOZ3qULYqvB5mKED+j4opApnc7IudqvwjN+6epDk79wA+oRsTfQv2GEvG08ncYp3gx6fdWNJjSMQOTAG2GOh0m6XBt5kJY9iRXw6UZWq7u742MdqCOHPvcCYuV2Z3Q4IYL9r/mSWqIDtruvcBVvmUJBOqZ5lex6F4Hsj6Bg8/7uu0+O5ez0Fq2epYG0zB/S2QedGgPRUNJvSBG3lKrmakRQm9dulcx01kx6Mbr0rHL6OZUyBglk0EhFbgOLnMmgob4m+PP/Lhs9Vg/qByLpEKn8e4QXbiUCWBR743VGhaVwV4lZJQH3xflE4hQVOi/KwZ65rj8/vhW7kiMSDx69VCZbG1ZqLj/oBK+UAwuWis5seInKcrk4ZOLhLXMirN2HnqnJZeMm5rt3aXPY5pmer1BQ8REz0yGVgGbRXOCQ+ueQoUxZ14zy5Z6pOqdvI/I7VK/fsX2/bbN7uB5VMd5qNA7djF8K+xj/MDBcB9Sf8Z71HnlgUXHxQhOrVXRNvgjlgOP8LwrfUHCjDxub+CZwwB+iaxVPUXHu7tWZQM9DpRleg4XiNaQ7OAWK133eBI0HbUO2a1auBA6K6GDc9slUB8OODMdTIW5+GGhh2DnQeh2QHidHp9PdJidkLzPCd/KMd175zric4hOPtIeVokHkbU03totGTqqmFjAk9G7b5Dhgd6jDDY1VynbadmI86WAJ/yZNNirX0yP8nSmglb2mNRJF3WUxaR3P+tLYUJRAw4KNupcLERta0wOXBoRCmSFth/jG/b7teMAb4kewXX6RFIr5GXR+2dJDBDI8dC+6Jn/9NXSQgSKP6KC8zvweyN7ow+WTuhk0mZYMxrEGdrpBNyxZsrZS4kTF5tv+zHMIXTje7JSLqdMUNVfzzOchrOAiJo5C/9K2T4y8lX+YlkSDIhK1IB46gUCwc0HK/CsFx2gVThln0zwAeU2TxQtCf3f+fnuuDVEKXXOM9re+tLmPf1ct7h4hbTDIwTDNpj9fR/YbKSV2mGxPc57CbHDabHBWwhGuN3x9O+upw1wEw+Gn6QIc5rQ3TFXvlDCROXEqTbPbWh0+wxGzwH/IDZjxUwMM7ouO9YhBtW/4pjYAPdmp5E483bFH5SWODPNOHWXfPJe/I0NveMSzFQYLIt58Zz4EVzhUsqwsjfFxB7SjeWHq2xLxZ5D1uhimXjTVKgWq9zjHLvhn+EprRMe9rTOW5uhTaj7DQBJrngietkEoHaBz7yc7xoJDDe20mY+1ewsytPSvuiaHzc5JtAVW8jY4hdRi1W/4Z7aOMtpbZZKtjo8YP7l/+2q3nlULaZkv0ERN+xHL6XDBJFOxtu8GtXA/2Zm/OUwlOhNoetdos31coZlg0Bd8gJp1B6i5UwLqGOnKY4FjdNDNEQiQwxt9S/fKvVAcxtxF0mJjdYLta7jYIgzUWfqQ+LGr7RZhWxXfkyUqOV1I06Wfu3Xv+ToTKtvoOuU2dDUX3e2kQHOxvh7lTHoB3T3sVgjF++5OqFRU38Z4950ucQs14rTWrw4/cprfdJfYU4ddxErkbsXs7dYKab/N/tpnaHfkeNDW71kb7L/6H30T+IKg1HYXmy+EdPpHelDVxkUncOpYUHoLwJXnaIzxmtKj+IOZ3KwnSwMDRyFaOfLN2mRxluvVB3LVbQkRUm8Yk9a0gaakVkOuusYCxc3eT79FpwQ+/CPTab4diuLal6RMKgGEIXJqGMRXUyf/Ul5l45mJnbRciYYoisnBTdqkjoZO/voyJd/ItQYCJi6FRCS6h3qksjkRKuWl5Lx3Uh4F2YDix+NIDy8WCmrasQvuyuy4gULkfgoIn8ktn7DgDRsH3F5l33CcrggXHbvzR8np6mejmuQtTgaqfC9WTHEQqDGjVN5Eb8cFUeV197x9498STwxe+55ETchFnNVzHcRPJ7B//1LLT+MRdcjhbyyl97xpUiOYffEQ+MnGhqL6ueyC2jpWVpWDPtRFaF95Zc8aHPw4nTDJrzPdq/VKmcgsgHfhszhsJDLDJ0xKIIDL6t50BFNTqINjpGTt5LUB2ImzEqtVTPxXVf19zUojRXuUBI7ZnSXjchYiL3S+3ONwq/lVW9ac4X8hMy4kHFPAJO5WgBmhiiBEoaPNypB7I07n4v8GiGVJNGGbG/RtDBbpDFc2tftWXVumqY6kx1L7gpUr+BGZspGkgPkXWkf5H0Q1AF57EYPbs4OnY7uQa0GdGkZf4n+C6MFMuFj7RpKMJdwuObPLKpHxWDFOClbyeydgdDJ2/NvTLqQRdP/Wy+stnIXNMEeALVNfPqjKxoSQZRCjq7wv8zMGsiome3eOzTmxd7C3fCfWV9For/e8M1Bcw0igMsz/ON2FYxjuuv0p1NfUUurqgz8G9Vdh6yv6FwfQYAw==", + "page_age": "1 month ago", + "title": "The next evolution of the Agents SDK | OpenAI", + "type": "web_search_result", + "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + }, + { + "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDD6D8gCUj0z53z/jmxoMvvotV5Cja2ew87/5IjDCIH1vhRznhHZiGL1yShZJLqLK7JpQLI5k8NWj2/7DnVf96p2SRcz3oSJXuD7arfcq6x0Oxsn02uS3i6yrQVmrSI+0EBsN24btcHfG2Sgn8rs0TLvYfS6mE2HH6+HfmZl6FaI87BRqq1aWGVMqHzoczMXDa/98BpVhDyggIDcUIlvkUln2tBKpi/6a4h/xliDVJXofPDeAoac143BkUdXkiOddbHwvqU2CEGBSYtnTES0kUNS+n5WNWd4DkrU7kQk+OpZ2DuCvmX7k5EaoSv8N7TGg31ZfsLcBFTw4NwHvk5KmYQnGTKfKLrCxy+G4Z+2+sWrBRP1THQ/JscDNBowDxneYNMflgMlcR2/kWEUB0fcFk7j9lxjaYYVsy4aD+3X/tC9cEiWbEQsDjdPsP+ndBFOKjXdBXFXiAga+wr0nXXUp79jFu8YpxJSM6tQ9YacCXD5u8UE8ZPZxn18+7sYdaJi89Jou6rcta6Ltsf8yIcJLy3EFGsz9aC1mMX+JCc3hlCIQjo4tsoPVQHYVon7llq3mN7Wie9v5JWRA3ce0nXYQ79Tp2uEVbkKqeh3krTqkZbUmvsgHI9TYdi4Hx9bMG5uhhFvHoSqzSgPTIjNfTwBWzocvrYfOGF7acTLrQo9NNPnbfzTJ//wvx12F38eXkt6dkMxRTy/RoELFqAUMC2yimdiZX2HQuLrmgoqb5BOTvcRzsIzSAh8hM2WmO+9DMAhET3jgdVxahZi6u3LkDY49OyNqIptrnMurrncfniemzSJcrTcno+tmlLb6y/kQs4PQM32u7mH6HFtN3yBgxn0uVmmnVbHKnYYiOZvz1IDsLLhFwIV4gVNcLj0AXpOvTC3C4Ya82X8ya/+lEPK9qdtf3zA4Nuha1/LMQsACb+dor534p2IKb4xHIQIKJ5G/Hd5dg8d4W4snZkvFctOrffjMtKTnIbHJ/mYR8GSPHrx8CR4RVImv3xHI/qltcHlijGnkEO6TA7IMbJprpdrCiL8iiCiGlDncSZbV8ClLlqVd8fZwPrgXKRp+3NdRtbKh5GlMT07EJ1mHy0CaZMAFly8fvH5X64CVhU9OTxnf2ML9lh8y5LGQZdj0UR7YKfS75XlClZBWHSl9v9hz584Lwi3arvPY7QMYoXwvXjZrqK5vG2YYVgbmcwR4jLpV+Pocj/srCX+6AD0r+G2/xU7fxFrd/kASTJZvAkGCoXcMwpOrGPRAb/Hl8HxTtghRhYpkDm1EYKQfzmbYkWqarIzBjLvWfvut1HOioh/rzuaXACcySWG6KwokmLy+LJDgNYBLr+Ksa+JV56T1BThazvp5JnvHKwf9vtq5j508T6cHndQljcgl2NVT1q4QEg3azvTtT4QNbBd5MmH8Bv7Qdrm38MoZObPyEGyuzs641Md0q6EQuBJD/JFmw5j2GhzqbDMAeCC6YM9iJuoR+LdWIDOQfLqKTtlxH1iEYxtPiYHEtwgA625cTxUhJ131xLbgZyI2Kal8i3qGehHCatU36b7xVHsfHDsejW62eg5PFVGen2qDLPz2O63wN03sD3wE2YkCFUgL+xgybW3o/QLtoxXGi+/QK3ow+CsIM4rdxJZ8A5/sOnftHmrCkzB1HvA3QT/fPYqjU71GIWzzuBRajDZUxyRDA1dWg7AGHr+6espdMmCp/5blwau5uXLBsBTskhTJcRXTbhISV9v9+EMniix+Zrs469cGTEv3mTHlVT5Xf9GXEWzbkrIWSQCc3EZIqddla4HMX2cTKKHKRRXnM0hAIH/CqSWx79bQIAwKJnl8AmIPGR3Nop+pCC2MPjjcfVszqQ09IDPlht114JCXXHj16S/LKGpo9Xp8fpmoO9+U/QprLf80OecdJdmANTkK7VeCeEP4Nnt46JhGL4eHBPJz030BvlzGHDMUwV1EkQO7l0qdoL8LFn/aSvYmweXEp3zgU74Ta3aKbr+WJEfYfM/Nia+GrfGxjkA+cFM0Pw7GmRnNn3En9yy67nnL79CoGbwkzF7MkPS4dY+SYKQVSlhP4xvFtkk+kp7Qu63yIp9QkbaSIvblwwu03SIrBedl3D2ORoGnZ8+UXglDN7sYnNF+vm0HUwYHb1gRuh3NhRzt95ZXOdbP/nmvN5cy1jfySmddYTjrs6Krt0e4gNXQGYNHAAOh119oCRg1VV1RwVvSHfB3EHhSQpjERIFcj98tZfK/TnAdDapCDZklBnS9yHh3kAfcCqXxZxSKwasNwXyDY74TzSo95a+nmFWkOlWOkVLMEA31QWuXg1uQiV0kZBv68ihaK34JrlfkqWylUGGTExU/5Fnv87g+osHtGsSU/qFRjhtCOTKv36a4bZvvodq0Q5VbxcUWk1y2iRiymgCRsq+JuJOUbz4zFBVpk1MewBUpMqD1fmh3cPL+/8gx7OpmOdpDjF3mnRetPjIWhemPJoXxPepa7t3i4Kj5CanfN5GZVwkvSwKabwBUPj8xQNEZjsGvMlU2lu+zE7QljtE6vH+3zeZDPJUjQMYW7mxpiku9fk1854330BzW9y4+SEJwHHrWWJhasDC7qtqRB6NrA3NBJmJOf2OH7RezjhW7HZ4gNZ4lAiIh+3nj0MOdvunOInzxAO8J18SpE5OaG4cKebWbd0tSBOEvInZJiiXCUybid4fT3y3EH8jzNYM8adqo2eMFp3SJlzSOLhDvXjEGrHZSSYAaKo0XRZsKc/PlYMe1KWvZaq5sYlAC3LkUwAwYv7k5/hkK7sQVCeS6+Qz5ts2VPp92RsEWx8kI3lxdiTArOFYU3weGgE0/O5yG3+HyMAmxRi2FA0YLzHLi6aH668K1VK82za9teJoXK+WKaZQUgWCNp2r1Ss4xSzzdKPC5Vri/aZsxW58Ax3InWwYE0Jh3aF14TvuvbG0JlB5cUSrl0648tMROO/LjKtdqAm/SmIz7X3OrL+gIsI4Hko5M2WuwifRMW+IQTDefMDMI8s1hcO1ZgIpGODzgQyTQkBY2TJ7cQIB+CrGSF6uwZ+a4bbo/QC1r6O5VuTE+FsE/S2Ph+0AMWtAJp1NrD9QsgQfduPCFPMZTYGSs+OsY/qLKI8qQ9HueQNlYNWbi1ynAAp0D1EzIXu3vjmFTYnA8IYLln/PDJS+66Zac+TtD79spj06iI6oPWMG/GFmEVeetTRHltEuLAO8bP5D5A8j8u5GMszAqQdtWb3/MQNIIeZkIhVxhW6WMaw0oL9JktjWrH84ynBr5DogETGmkQhhcWay3KKfheRrZP3kRu1tPs4TIdVhuHLx+3XjxiQfk+7VEVSwRk2EDt29DyN2/JBxlGDMnzy5KhO4Mp6WrHks1kyvRFXsoQuWDm2S+nNm0j9OL3cOmQy5f9QQm8RUoK5Xk2pv49HH2qraeWfD2m1a2gOCVGOw2FPzgSsg6jSPp3vzC13O64la4hdZ1KlAX5vKP9EeTJeYIY2Xn7Vi87UgVMtrTAB9Rb2OgmvDYI7xPmI7eFfbtxpdlDhOvYnYc6FzuilHPm0uQ6cu/9b7xl90K3bZQUSHzeEgDNbeCFzMMur29XcN/A2G3wjEKby86Mkfbbhj/BzdD0RvYuHuWJFOO8dapxulPgjm6FJURcnONSHUS/+p3gTWKy5HFJitzyvpfL995rm5hRRNHtkl3b4I4sx6tfr6Fnz8p2rEGonIPgYvQT++O+Ry2xKCLv5tIF5bShdG6zsrhf4+JkO2g0UXvYGDIZd0Q5xWDivV6m28pKd+bDZ2eYFTLk2SGNHPJBdZ2dz1RDOjwuJQvx+ayHaI0PZnc+ued663wScXo0j8dyEfbQ3u94LqMMRNuHP1twfzBQF42TTxRUiy57q+xa6D7BXosoSdHgRhGvCyu6xYGCxVJq2IzXfpqOn9wC6D60uaMq3FPoxpZFgcFkxZnhdHC1rCq+QWA6KG+/kTZ1H1Ol8fd64/7J1UdiBSB+HNZBAj+Gvu91N578mhK75Ipkiy/hsGpXSpuOFgcsR2ND1LfpB96JUJ5DZPhzxp6eENjn395037Sr2nwtfo/FhYkOKQDoG31HC7mB2kU4l9eyiaxJy37l+LZzn8BpuO1kwThbj/NOPNfm6EUSmCOxAcQI3j9oJApMkXjdU66dpBSJ9mr9JR1eXS7JIvluQXT2ZRbauoYPGi/vyyhxQ0kPV5liu7d31DUBD18NMK7mFpbtW1IPFSdQ8Zp/iYlfP1l1tfdC9qKRS950P+p/tEuSFQDZgWNh4hA1UOwSEFUxuBBiXVJSyGDv8FwdBQ8kCRpvjUMtuAQi8LoeriS+QpH9vJbCh55a9NAJ7H1pHhfhNStztjUNKlutX6WlL8zGnMP32TMQz69VNDmuJFK87PkrqiXTzkjSTO4K2zlhnlDw+hlthi+plNzJTZoL67bjVX5wlXs1ulUt73qa+8ephpnABXopPAl9KwdNjdSYTyWwIk7gU6HpQszOGMbD+I6HGCila1Iw1/9eW4mWPu41sUOMqUGxJwb6rWZxnIsNufK62/KMZNHgEURfjXNvXVupMTdSw/+y3Rj5XFwhOhT5MhbwSUAsAFsLMi+CZoA+0ewIOCx/rGHlxta0D+IXUZQ8sCXGotam6HO3plT7WH9u8CjF/J1h54474ajd+WtPH7OpeElaxrgIjxTqsSw2IfUR3LJZNLY7YQocwu2aIXtQf3jxAxycCYYct7v1Yi1G3Rind56JqAHE9fOPEh+NjPmX1jS+KYyENXaefHdDO7V4f1TDGnb/MtkHnVQeHF9tGW4UDPm98DbfUQoJOZJ7+psdTVqocCAwyldU+LYFSyZ/A6Sb5PPtj/kzwdRl9hXwV2Li0SpkffmaLbk1Iwe5iC6pusq2eqXWSWgCJbnAZiBSP6Vv4B6XqwV87ei9XV4ePmjWl3daoqsLQqhTXzQPFaZtSA7/v6L3Ort/O5IU4OdFjPP5IVO6gbKlvRyY62VSpJIxEfi76sEjXzNk+gv4VEZ8jRfvtFNwfjGxVxJZ0X2PfjtJKLiNNA74ieuCkVUkW2HhhM9KTeC0UqQCAbP31mh8omOSPW5DLpotSPsvduLZRJee/twd1AeJ7uGYjqX+iwB21bDwagLfC1X+/BoPcwlEn3BoCyaB9S1JGDyP7ahyFytmhWpgxEUdtbkLl8o3Cxy/ebbliGysr4v5OUjAlThZwhOcjoPZ9nRuJAYAw==", + "page_age": "April 8, 2026", + "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + "type": "web_search_result", + "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + }, + { + "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDHp7NHAW60Zr6H8nyhoM9qwhyx9pbTyulCLyIjCSxnQLBgwDFHz9Tmwddm7CEGWVaTnpl2JXFhnk9GxHCzZBP4Yi19O6FghRVAv8sNwqhxG/+swA2faHCZ7s7frNn+z8kSdq9hyXDhNeTLLr1eCu0bruQxOuM+tht3ZFYdb5hj8/tOIswNxRbLlT92lPeg6wRw8XzUWhMaEb5aY/abhDcIvRo7JOog9TWzvf78al9WTmRua4eiLgH9jPfNFkY5LoNxe5fW86L/B4M+vgwV1jMdkfdrj35vp5CvUh9VC/rqJb4W2zZ5DalvR3wN/EjPUmmKOa8G2ZpsZ5FVFKHhl6SOEP71IdxqTolZXrk6ECtPP4612dxP0qLLDTbEnJqw8/Gcjrb6UrpxZ2d3lZu+1b4XV9FVtFivJ1T9A+2yjqs+vA/kdFaQGZMkxhW0+gbhDfQyQ1RrAtb5eOu207NaZJOhw8HM98X4MMnZkVllm5W1zT359pV7zV9aXTsowkcp5aleeFfA25ChIaVW3TRhQB+ra+aSk/sm5k74MfNoYw/SGzuHUFlfyeUoRNGG5Wf+gJvoOlBxGnZcLcKRyBUGi9tKIP9dvt6AXrhKyFUr9qMBfVMS0VSBZmKKtzFkeF04KmX8g0kHBZDm/+CNEIVl7K5s0DzhQ+aI6PgpkAG7mGAGUp1tg95x1Hr+JQgKnKY1TuoWWv0kMoD2O4aOMpxBFyvMPBaLEsTCpnkORwNi2EJ5HbgYSbw/6BrWpLjF6c7M4/gtsX0iFsVy+mYjTVw52cc96BDbkXdo6sYdqYSOZLR+P+XCMnK6dN/r4r51OkWZaYAhnrWGMs72EfYmYKIyRrAMJPsjlWXmzCIma2tZTKinM2Zay5EswQbF5tmEwW5iMUv/SkbHeY7JuthAXMIDf/YahazZhfL5+91en+HMFFGC7BKKH/FZP5lxOUNYmqVgI8JmRcOekPy4U0PvY15zIiYNqQwXj0q69ug/fDT9MFm+74oJrIUj/JOY8+Ye2nJlseIdcQ4wRPw2GLBovTuM3KlJy05MaCjrSSAK+zNRS6pSBWGkVEtLfLwyzOMhHlb5t0gsGQj+C7FAaqdADDAIwJAUUhCtWizFtOEB/nHozZCKjZVr508IEbPN/XDE1Cd6urTgGfp6awEHv2Urs9cx9p5kFiy5lWBbXy2OlxG/J0hN5N5+pv6Ta6L+tsmRhp03xKkPWMqJQlgM7ZNNDS2d6W7uRxE/cQGJ9Kn6YTqIToH9i+3K6DB+8r7Awxl0S49QEOZVgajHu21LOJyOzNcLwO0qyLlCq+VTj+q6hkQPhSGF365WvU77iUpONEwvSv1paudrVrTxCxVkg2qWSOriJeyoGxr2CWZTmcCVG2SKcJCFMitMrbZY0UsMPQBh5aRWiiJY5sDEUjCW4MwpmGqBeQQzJvkNgXuT9BhTigjNNVaQSx4w11JGbcYQfFmVLvslQ/uPklViGznUs08RmUS5djSuIdSXqKjc4x/YDBza/Cb9u3Lz8cv6FKnvxAfO4vPzw/k+5ie2QWCl0cwKJDNgXBId9178yxvgzPSDUp6tQw9OPTNnZlf+fKquAuofPOyRs/IlY44OOrOCh32pfDYQpWFmfKnm1N55C95osZjF4b1J53+dkJpC3HNr6W4vjSTqje6LoK9Qgud/SBsRINk1/45YRPOtWpegZgWfLTPtK1ki3W+U6D24a1H76O5ok/TUZIjSl9rDwpsqTQPkO7IoOXkQX5aozDGMRVhZvT6fr3j6pb8XGH/NZKLWd7Skxu8zMMxZRdeZpLH7XoqAYbeR/hnG1jSs6OUshFYh256vGD69eMfPUvoLfwoVcgqyMQDOHlyk9wmqtBKelsEB7j6h6jkK7rFzxMF75KzU30ObmY3IF+aeom1suL/JpuAuao39WOSzJUeUc6TuQQSi1wL+RD6kwU4ECnzEjeeJ243fijckmsMxJz/FK4PQwbuffEsx27kt6wggQuQwfGHi+G4L/FE4cRWZV6darBkFNnMHvqE42QUaEwxLu5h+MuXhoaIys2MmakQj2Rg3eSPM58XvLspSCVE632nWXKfHc4fOqzTIQ/kOVy8v2X0Hcpek6y0AXktakRTjFP8KhjVf2c91/2QfXvcioG1xx51TH3grDy6noKadu0uFOitxR288Xs4s79HKd9AY0dlqqOb+LdnrMfbBIi0ampKVGnu16RRcjJ00w7D5p6XfdRWHyO7AP9Vf4YM7WzIiub0Vtp3Z+MFB2GIZeFPWeU0RdQJH0MRBG12Psnp1Vn5vjLQGt9l2kXsEWTm+EIsRl2BvYKKCUkFzCjmm94nyPddvay+77mPXOuviB36A874vlM0b/0KpcPVEKTdZRwNFVVWDmTiqta77Cey3M9UZq1KLHsxHk7xqiuKlhze8SOVilXNK8q1oy1z8F+dbWgbuCdq0Mz9kPvCUqwATNjNN7eCFKjjcNxAva67ZsporCi9YsxUwZBC1L4wZiGAwK4NopPCkQmjJaks1fjWiVLcrXqIe54M4HvOgcykIgbT/xMHo20waOllfJUNImMNgFLDx1tKZjRQ0xBCedwnjWsrJEHn+7fH3aXU04vP8DROwLpE3ZRwYjvF9i3uuTLL8fo5RTtO+RaOctKjruNi0xEDISh1nTjpBZj/qmyASbUD4DOfqWhUeNrJ9Qo/vN/v+oTwQB9kCVyVxr58U5uLEMj/s2kxmx2sa4hmuhen7h6CV8H+e+u+BcXUpZ8HXhde1vAv6VNp47PCdCx76ovIdsfihdJByXKhJDnsP5XwCnvIy95+0RzVzUJcSAyLnbl0VKPwqZExVDVbN2OVlDqrhAyCasD3A249rqTyb6n1Ng93VQyT1jJ034kJ2XlEqvUvIsNGL1jDZ66aUEBQZuKWZMVBCKdDEPdK0zT3ZGKl80CVdbH9RYMjXs+KbPk/4nPsRK59x4Sbzacs7e2UwEaHf79UizxkPzk/gfyCkbeD+qK0f4hbUj0asfewKnhPjdxyJ38rAO/0hgD", + "page_age": "4 days ago", + "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + "type": "web_search_result", + "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + }, + { + "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDDXQvvZiaQx5GuVS8BoMuqycwt0jjZsQPxDzIjBI724GTF88TbzUz0t+31OKIzUXDo87rnUezvuct27PzRnGXk4sGY6bobiUrjrIzwkqjSClJ8gbHpjMg0E9usP6I1c45+WTdRESXalxCpRdcDDy8O9xU6O+gatvKOKg9lf64DKRP7oOgB6C606hPDHlLXtM/fOrg/5cX3dUx0lXvZ9FzbnhAz+gC+pE02JR1LgHAMyeGY0wBEAiuSaLH7dFMT4/OmxzG0kD7UB763T3bJf5iImGudUzz8EjqS8jDX0K7z3eG0E7/ifIDPW+Jv0mm/Bh4hbdB7DNHD4CRnw/Yc7LAlfcmem9+d14g/BVYzvKk2zUiNkuaiDc4SyL6RJ9CxrqKOy52ym/frIdqYV07vEekPFZbqWtF+YG6I6YncpibHauarxjBga9WluAC28pAKCFqdXV2XwnqPORsczWu3Nv/HdboSKRRll5NE9gJDCqbr7MRbPQJDAXLMI8NEqENHV+mSH4uIHrE2HHuwA5w2uXOLl8CGxYMRc0xmCo9ExEdteG/f6Cz5vEcrjC4K4MIIIQC3UxNdiJsRGXSCELH2HkfF/enMYcpshD2vi9UqTCfCZB2CE1O3zIKRWZ9ACTErGCaShnH8d35SXzCjB8cNMIfPuURqbJXuCO8lBb6UGDgxlsID/9D7bk2Nxw3/yW517jYs0m5cC0xV9p7Q+GzqJwi8G2sKH6VXCd7yn+bwlRKHBuBpVUAono4pnN1dccMxJ4Mr9kOsjhqTjPNYp/nn9cudQzGu+YWP2/z9bqR4/4ytSVzViIsR/abOo2YgCTnjar30OEt5vC9K++6YFFQkHUV5P2I+zVAQBUOQDxYDp/WR5EKa29hkE7OU/PzdbHYlsQVcbT21lISs2uIx+MolI4c8Dw8W5j+Ngcea3MB/Ozx+t9Zsk/Of0cVt1fZwy5kTyNNTyl7zjzarczyAD1w40VPxsX0yf+iD927xUfTECcor7GcA0LYSgMiVUQSlu5UBrduWZk67v+MqQb99RKTpVwGDUlQpP3sgvz5ivP/ASfGVUMsIwy+dA0Trrw/itf//c3/fHr+KGuIcFOLekC9VYJOjrQXx513yXF2X/10aYd7skzLBHolwy/93CrDjCPkeVVFB6NkxefBXMP0KQI1aFg/S9OY0vQ3WLbow54CZE2AtCP5lcp5SDAcs+2j2QWwlCjcYVR1OhDDSLpY7Vwy08ZL+Z3k7GBVJKj6eeHguvuE1BrGPOZ4P6EFuhA5ugQ12RWBEdpMYFEGDvBw2afTwj7mLjsKpHDwu4XQCp72iEWtooFm4n2TFOmgl+3E+E62P2V13TC9sdwisK/iYBdiI07wIzsVJe2USjLLZXa0tImb4FCVVKCer6wg8suh4EEyV+Ehepg8KCrrhTX5uC8vn3OaHSsvbEvaiAVyETP1cthM6YhZJrzI69Pwpa5jHlonLEH6Hhm1p9Wxiyn8mVLu43JQyU0XtX1+UkJyaaMQ0R2RhRT4MTG3E1U+Iz2FVavVEuAx3qzdivVg/weKDv4gBSYf2Zvfm7pHyCleJ+0r0LP1ZwhdrqztUuJux7lNAUU/z5vFSHbjeNjhIDuR994oLjYMRwQPQ2IJ3aXJnNXYcJuGQSEhF3/crj3toekCjQSvZrud8ZYHxqNQYtvld78ImTrfuijuN+Fr7xPzhTAAduztCnastjLmcsjiwEUNKkfeCIpvsanMqAJJbYsYUxxFgh37klk878i0l6CN33qG0RwLV8rWUMCMR+H5Q0Zww0zRPWGjzTKBELVadts9ks5K3dNmm+V50kIRyzeIF6DXmFShDQH2OA82hjnPVSQwJk14uA8acPhJgPfM1hoc1iniGv9MtkHyS6G3tJiz15/h4zsRhIy4+O/fKDcfvWW0YufkAOMxoboGans8sCS3mpOzuwgiBUkVzCwihsh8h44lpLo/f7GCBSZPlbu8ZZDn9nMXVttEJJKwmWQpzDv32FXlsOMmI42D8v5++7A8ahai5RhhCewHlh2dPAqR1iD35ODFEEsYKZvsDEyc5y2Jm4OBAcNS76F59DYuCIDB9S1JeS3u5KYXuTBlbx22xpV3MDphXqxJeDDEwkmsBbWExp1W8U54ZMhsbIPeGbPcJqOFBQfIC+BKHCs069WMApP3O4bnSpTzSlScGKDWgCCGy4pJt/eGLtUJCwy92NuYlCR5LOCTQ0g/SEzef1OAf+zd1Zz6cjdY8aWEBD8ZNAswIQUSQbsnwgBg6u0qU/Snv/MirM8nkxUDji7wsYrCjhCG80fz6sv0arKMW468/GwlvCzM8Avd4F4oqZjjjEvvh+bv1BM11nPT52pYGk/jVB1b/Q6VCUs9drlvNZT4GmIcmARPfEBRrXuJIcEQO+ljHLgfuM5eBMEEjgFYAJiuZKHw7tjZqvTVMzsvnnS+28He9R7MbsK5cL54jcE82HQFs3XKS5y3FU+58BjTHxvW4nhrZfKvGtB6nkuaUhOADZv3GGNFZdg0kICC2m0XtYP8oOWCDNnPy5Nu3GxJLKzx5MLZiPkBAQKsfRVFLF0t1Qt5oymplxhZu+8iaRgG25SEhBpbF/vBuzTs29JlRi/F243RSOxFiFgNXqX58uruRu2oBpAhssyGHEG4fG++JbKuot2XK5fGBq9c5o3U73dEBApJKFgpfWvefKAMzsCMGSBScsVtfG7a34j+XT+tJKULIg/9f/b/NATbPT8f8/0fXfULUpSz/4FdsLJBWwNE12byCRLKRAZ5rmTpIfm2JlOPjyD2dNWmma/iHRyW7A28DBrhxRutS+VFAskQIm3yojHBBjk8Cj8z4MmXK6lv7HyGpgVPwdUwqwss17LFkq8UuNuwRvTcyrpdksaRCCVLSbQm/8Cc+K8Bvx+WkPzVuqndkQbYV6j1tB1S+nbQwRXG7oMvaBrxOgcdxnMDLTZS50RbuLOHlc+49y2Lny+QOXXCFkjhHtG5sSeYfch3ABHTK6AHoOcJJfZtMq5CnPaLw6Tjb7hqvu9f64fzgdCGelRAlhgmtZ1lyTrM0k2hTFpm28jWhg3Ht0v9OxQYZLEM+KBpRPA96QndWENFZ/NsBvF4w4085wUlHm74rBPxEu97FSck508kkvq2p0uXY89sL2eae/ZvBhmbUDzZWYktS0A7N0WuCEzE1H1vLyqwd2kY44n9QGJoncj1owQtWJE0Gy5C6vD5rQHId5bCGkhn3DY8YKLRKmblOONp863ydD+yeWcle3WPb43S5/W66Z1ib3iIa6OpDde77DnhGlbbTx45ErFW0CEG8v5nhif5xtyYm/dxKlgjZ4Yt8KlX3euKIdr5U7x850wAZ0UqH6Hc2aCDJ+O4Bx6iaBYnfFNm/mby4553FCNogtAsC/Lx5jgqdG+bkaq4imF40G0bxol78pkrY74j7LDrrszbQNSvt7KCGJAnJlh2DHUWQKqCPiH+UfmBxJg/4ROawCCX1EirCF5aT/DN1Bv3NhJ7xhhOWbmGnnXC3A4FXy+tFwPqwQh9iZoupEDO21pgmtsvOXbiPP/WFQ4//GQJOeZxwf+0O/nJPKreh7JCV9H4/1UbU/ptgRBFByp2jJybIwEzPTgjTaAvRH81NGHMvkuo18ejaHtsqDt/hKEYPeyF3hRZ+LS+MhYnTd0NA0Wgp5/vPXhT6UcAijmHi8/lk7DL6HJe57q9Ur1xkJN7FydctGZIk6ZXHhWnRSzyPqsmYoibZJ8pojpoZL53VCnmFHGZrc7Ty+yNtQl+ur1CN79CIIC+wWJ5zV7q3qkrWCneG6hfq4HVZR1Hzh/4wtdV11+oalMfYVDA86gpdvVlCzM42mVMeYYoUeLYZ30VLkHo5J8qTeaGoiEslhEpuk9hY+6OFgSNXoXWrjV35uqRHUnS6aRAwgQcmbbuykZguLqy9axEc6Zsw6tAzk+U4spoLlzsRwpfYgqrq+ykkKgeZ6nz4To3bVfDXfYisXW2EbLp/jq2eq84TZcgsBMmj9kRD4ZeB05WjKwEsCvta7KEZgsWS3wTW+6YGwofdFhXLMvmnI3yH8fOxw4cCp+Afor+6D9zFyO+ABaAKTOY+oHTidrLLEjuh3qxxfz97YmeX72yfhP4Aci/SmpiZLbLS1D7DJhld5ocZT9IwCIv5Mc3WBl8VmiF1SvWbIzgI9cyS0JvNr4aL3PwEDzeI6MZJxBLJGbVsNuKLQVqS0juuvQMPto52UiKTXDC54VgCAVTNlqzYBFVMXxP4qIKrBNb3VTxfXrVSl5pTn3pLIOuXaLoB2nx9yRLHP6W9HVjoKFM7nVVZrLlD0RmR3MeVdXN9b/VEuWW/aE7bDJREPGRuIjYS8Zm+rzKWywTwjfWj7AVclIVEUXOPFxJkJzU0qV1uAqtPwoBxAYKMQlqeXR5NhVXmWOmW6ATBpxkevhgkq7iGU5XwlEg4vrgowtlVxwTNFa5FMNZsH/ZtWkJaiMwQryHe+33LQGneDU5sWAZfkEptY82FHbZ3flqDL9HGSRdEOOGA5URUuIvFpypPcu82JOhLA4TFn3vR7n8rXFxXwHN6hEtDoHasJ4rPu/X9Id7AiVkBVyEqDWm0WA1GnNufBXJhJAmlvHFBOrbhPfK6RmbB8Dh1KKIPHCJrpyAre1tpEdnmyryt5/wohZy5glnmbRSeHMoXdyKlA2H7PDOZ7SyVyv7maQo4v64S2G50zB4R9hIWLYEN5iamYyEZOg7Q9ZZJC4RE8Deq5flc4hPao13et36GXt5mgUadbFV3OC/mfCJ4zZiAgVVPDwmN9RYIohvah7H5+P9mfBdwPQNsol1Nhy1hFLMxN0tVfNvV8j7p7CuzsXk2X0TXW/9Ezv3GSPx7kbQJbenYYEvtkuNEmsyJYFgvAy+AWNxHcMhw9bRatpv6E7CW5BPbIjevHvrlLWs6u1ElxADFAdnhOz/J4k+FIMVUwakgmyv6GrB7s2Cj3u/+PvL2VS0XQV71YxXJDvPIlGnXViVOzxkY3EYnBaVtOxQlXhZ3HmYKourd7nzG85FAWYaK9vxyZeW+6i0aAzHDkJvF2qncCY97fyYGSatF9agGhR1m2MKcVh405hxAzBR0bVVf3FMcQYnczwJp8fvD+Z2E8kyqPln40byTUZ3iz1FiMh4xA2GvNhSwkpQlIZQAmoLvUwH8hMB8GlBtGM5b/B9fqxSXqLXwjGGV+nDC6q2KMl2AcJHCHBF7S1PMcJs9EeCoO0A+Ff184f2Wz8DtanjiwF/18qfvA6Ra6MXGPzspC+CoXpz3BNAELNb0q48I8dbfI0gZjBmQJ0XtI6jwCjN+e7hWy8g9bWOp7/xMrlfCPapjETKG7SCzR8Kk7wY2h/7HXTkBWOJlGXcwTDkWf2f4t4OG7VHb2kmMRUN5oPQxdSeD7T7f2/inVszGCWkd6z09UPKzJnLvF5s3R/p/JPoXCWorORlQxOJOrkqgNusMb8yEPi/5YJOXbzp1EomBGdBbOPkDl67/1yPFXxqiP0l3y7CQeCgJWrSTz3w2RUi1CvfGPTBcc0fIQ1F2WZ8ox5aWXmUMb0LR1ZPHigQ6GUJW8TO4RWj+nNKvk75hgD", + "page_age": null, + "title": "AI SDK 5 - Vercel", + "type": "web_search_result", + "url": "https://vercel.com/blog/ai-sdk-5" + }, + { + "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDELZl8TcoCcuJd5mpBoMP9vXxis08AkGn+khIjBqTvZQvXQKW9xVvflZxZ2Ieuhq5UJqEKMpTnqt5PRIecKBdF+Fhf3haI+V9xEFcrIqyg3AAFeVii7epQXmTKI/OIX53ha9Ovlh7gmiNMzr1PoIVTb5gDTH+54lA+CZOpr4jntYQKabYZSqqA4EpoRZKitkUJVbOuOdAqLlZ8UEguig8OQTSY5I1hdzAZ6c/4n7SGXNY2Ovn42TRlspOTnDFUY2UPpSz5fAfyradbr3fpJcR6BIXGNpwhQsf4v4rLEijOAxUwjzD5saxJPRizsF62pgCxJzHb5xeipw2dElxa6oFsBqFmN17NnQSJJC2xmpp+zWheho0nQAPZVrHAn1xSLHVjhCgPGmJeYsFTyAANA+5/WsqS1zzo6fWsmPbD1+6FSPt3EVYJ/6ATD+RedTUmMCjKspFaMxiDwop5RwzEb6R17X3lGxTU8V2mre4VwV7e0HjoHtKllbvtS+jqwF+1Mx7kufk/y6Tjv8LRJX32AjhEg0CuG69jnJ8aPPLFCB3GpLj1lHtucIZt2kQ9B8AMfbTimuRD4ba69xZdBPBR4ti6pV5wg3EhufIXiq1QV7w5ZmXK8uRkBJOR9v4eXLuQkn4P7OpwPDGEXCI/UrOXjufvvsMIX1abQ3zOpnR4hJ+xOy5C1WdFWbDOwHQhQcMTRNjPCHkQKYmDoW02adpNApxMOsDRR4Q5TF6WPLtxptPcrNgy9AWR17WrjZR7S+WZDdgwKSVM0ordVn4+fpd8hpDNW0UblGSVfH5FYA71BjG7JeDooIG06pSQPwTvIAk4XqCi0gMrn52r4ykPbDYX27+cgL6Nhe18tBCKk32I+9uxai8dlcYVdII9MSEa3BkdMo0uqwGiR7D+ebN9zaQ744nmw6vFCmw4FmkZBjp7sP0u7npMijuy7azeUjKx3+XHulJ/oIOk3eFgg+za6sjCGlvvJU0/mTNw5gctRd/WgCinpJP8rq6Egy13qHMgq5KHWejdZNuQR4n5/8tXvZ/EoEkmCra94avq54mkY8xF7/Rc1JrmKlEQXg8lznr0y0wX7vDdgVYgzT5SsbC1FvhhWEZ6uDVzi1w9IU/LY3UGkiGmxTkKfaaph4oNFRjPJKEvuXJsLhMA4FlM2fXs2g3TZcoySMrP5BYPIqUx3uZaXVdY6FaIoBsBttEpT9AnMJpFoB6mliLpRHawSpTN1xhvVjayWCgyKo1y1KDDOW9RpvvVwknX4nB6eqPBsZUZyGCixbAHvtAF0+Wl9A71QfZliH/Y6N3zNnQ+VXA+Bmowdu2ftr2H4hBX/lHTHMc+WHMIZReNyxP2uCryVAoI4p9Rfl673CTzywizbiuEF2clczrQ+wgMWlwImyI5E/lwUYfB9il7gpz8BoOhnM9zOb8qAfASuBvIfkU3G80UCMvV5vK7t0f7QDLtRmHlsqXMiR+zNcwRT3ciuDC2/8zWqW4f7MqsR/l8verPGj6TiLMR9ZqafXJ2ciU0s00HQpjSv/ikwDn4Lrgx+kCHo+ZkLHWPDZR5IVtRBI5lKoOX20D6x9iPtVihExH5zIfECCc1Qt7nOGr/HDp/qIe/K9Me20Ys7mZUrOsF6XrXpz+O8FNbq9LP8qR1lBHNzZBO8IOIXYl1wVEYB7ZWCno/M9lfMGPDYnh4VKbwJy7V8KgmpFpq7oUU9uW5+sE2SgmLpRnYXDwh+egJK6QN96O2ts4qPFK7nEuz3ymV00meDd/btZN57tQ78hioMPddXnvgdRrqcjPmQ344QXWLA3Xl7BGpmOClfx9mJ88RKndYtoMgDFsmeSpTAFCCKSXP1MGGiaFJC1Ru8BBTB3GjqZuN6AAHeTd4c4JgtDDE+QqMloF8qiUPCYe1T1nU3vLzfW7Y3Ua6pdGr6jJLtwqeL50YMgf5m10u4z5VAdRNb07bM+QHXvdf34tybOmsliCvJ2PzFfqb/nZCf3TSrf9Vj+Fm0Ddqj7b6ks5pu2QdooJumjaLWGdHBmxEFQ/ZF2cM8am/nbCPDCMNIndRioRV3Z5kJSIU7OniO3/KKmG03iZdqKDcgFY0E/yiinJ6YgiarLavoCGBwfufFcrc/IacAly1a2WxxPjbX+Qx1qSYavLtaOwIywTAQv5qdmt40z2DqeHwdTtBynY4zSpSse5Fvz8BDSN/Z4sUklFQsql7eTiUORS8jDrONMC5w8haMAOrzFiGfkYxd370ucJkgIQ9sHzhO2gGdrN17vjgqDY8Ct0LSxlDqICiK+cGgmiM4WVR/Zn23FXsKVUvHb+tbf16MExctgI/Ctm+v4n/p8KMxnnNUB5bkDwL0DPSEpqDoqXrcYi6kP7hnJ65UgSVnOf3MMJXDAIeY10QtLx3H/EFRomr+CXnv13T7kGVrcibVz/J56qr+uGAM=", + "page_age": "3 weeks ago", + "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + "type": "web_search_result", + "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + }, + { + "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDIPCrn2iOj697vEAzxoMTyuu3sH+AHxv/pXxIjBiz9d8C6vwLeSEWqtDDOZrRdUd/gEOL3FUUEXEUQr6dZnSueIVeSXmMYYULxALo4MqtRBiFWgv3nNsPkKlaHauMFgEgR2i5cKZkj5pXvh+XYgBCXOMaZKs2ZZZ2UpiJeCh+MEbvIrv+5CS7K09Si68atIVz/wsZNdJm+46s91+OR0Wtcrb3RWWaPoi8DQu2WneQwSSIXr1GcwT84nB9vsjQAhy4ccR35I/ZjjwWJ38unpAWmzyDa4DebDOPkKqd4FG/0HD+wFEig9zY6k38MDv7Aj0tpEHqdR3yzsEzhPXim6ZVL24PUef26/yuP3qhsflTaYfbBCXhePh0dDH6j+oPRGFpzDP+qNaeSBIxm7Z1pPvwa68PAD1WvR9QtAaTkBj22ZcHtEuBS5KruEaThrN9IIppwDXGLgMPYGROoeYUDyFFxa2n2t0NejmNXULIsqwF+oaf+UiU4DOJFpExZNpswdYCSEL0omhIq/P0O28PbYr3oW03oCR+Ya89oII6MDju8NwIpgnVWqgVActpSlJDV29Z+hZLbKznWjHFCS/QmQ14ph/iyAiz5nX6v6MVZdwsU0EemZzBR3Viat0SwF/A0bXepTspOiz07lN/gVDe6zg11eMVmm03IsJsF0+xcwP55ASPQz0c1NsIGw/3YUGltSbyGV2/QavthU1PL77EJMvqZIt3nDnFWV9yLNsl1bSN+Q2F3rruXLFFXIg40mjTbapynC2NJBTt7WJvid6gGJKYXSDO6M62/3T73/bO2tK7mU6XO6h345PG/uGIqO1BP+KmiGJbtAwXmCRMVrCwLo/gRxngoHRGftCFbbsZq4C7M8yM7+sZsEdB1LdjZVPHtwKrTiU+cYw62Yp8s5UBu2qUgqN5VlGNUhYkwFCDSaUvgBPTTY3va4qS7JabwPulOgXisJSbfOSWhxaCdH6/SHi45TCbta9RrF1TJI7J83k+jHItSU7XdlFJbgGcc59150TYrmhHkhpxJIJq9w4/BdpLywhTPrurNvlbrBl6O7XLfLEikpOgC7RJUefGhoCpBX+JB+33BROXTenMM4FeXVud4bP+kdXi5kslDd3XQwgn4Td3zt1D+Z5aWevIY5UUOaXTYDEnooxKH05rbb20w5XGqTNx1wO3+U9MCTI86vS8ZeTptu3S+EcHfecZLbLEHETv2BDaB/0TkVup+rheEEgP7qNxpmKarmvMXB9xxITHzmkAhjUi9IGKMm0Pht8Rtq5mq9J/2Z0iYNte7nFyZPTEc0vxiCE1VYAo85IsFcZHy/nObh0xvGXPWFVHj7LkWYfngxQ9KsQOeOaBCptP9o6d/XyLuxS6bzfx3KBTNlxIcVC3I9+i8+HIF2Sr20RpwQfBvefd3cjihV8HHyFj4C9oT8InlFsnhBzVCC3OTuwnCl6bfcRNuAz4R/8dwItMuBNGEBhFO5V/eHaofXfzOxriuddAfGSH+fHtpBv76N1AJn/n4lmKLNOa201hUBLAvyHEL4KMj+SJgu3XzkXezSg8swfWQtj/n01tIGFy6CDkiANmXHWwF+QN/IUrwt2NdtBVdV7ufKgiRWZOGvb2blBmLW1aCyTto7NPbLpKIpFk3Iph+TRWk8nHep3RIw+b3UT4Rd8lZgiPHTZiNf4ewWZrlfj7V40zQdHTX/oLmm2IGEMpymdTiUsHP/TBc0jSZl+BgSo4n4TBktOooqsx3StMCmPbtWEVR10bQCIbI//fgT16hvOJOjjmkWTUExmNf34PbH9Yc5ejs6/UAefelDupEHhOQQcJ5Hl7y/LqjAV+X1+7eFODTCvCEjbWY5pDiHwwr7LIyHE+O7CGZatdKISy5zUFFNy2wsDsjpjwlUPNwiUFWkWp0hMuVQabrYMpEhZp/2AeV8WoqdF+EdoIwohcALkUvLc3Tyfp7iTs70QKyi1j05r5sS18nkIQNmKcfg5cQO1jIFSYqJuzZ/k1N41tpZWGN+nZSPcMbdrv0GxerSExGRa5c1Z3n3w0QhvxdDWeEUQ/N2Glh+5Xw7dRWYLbeWOwqM+4xY97eCFcVKNHc1DQTAUf8oGQAEYuoxCxvUku/AaQoUbIeRjkZ2dEo2HF9EGlcJqihiNRCkOY1lB+Y+f2Vk62hzBbn4y+G2e0AQxeAALu/KaTAJMFW+tRSRYb957JgOncZ1Bt0eimOhVb+BcZJIjDLV7LsSsyr4TAnV3+pfZoAwrAFarPm9HzaWyf4hfB9LjC2shIjDWDlPSEXlMWijeuIHtZw+wp5y4DUq09dmMv0563aRxaBA5uCqYx9hFq5BaqZYbhcKavHB82GH2tLT3BsCMBY3yEdaf5Fdt644d6cCCPyr8Z9gcnTtnsuT77xyroW2rizHBuGv5k8cp8v8M2jbnBnoae94WP3BjO67Uc1UPnOOEZu3LZyAPAPJymRpPCo+I9/idLuxLrwH7Fv6JfzaJResFm5TDH5v5yg6Sear6LlQ8OJfIgFkTv5g4Q+rSFx8KqNKOCEHJ/GGEU8V14gpONoti/DPGM7huttkvGp5ygN/hlw6ICeqNF+q0tquUoBSNKtZY4liUH0a+zQduf9tUHcYbq6yYHPYPHQsL1Z0LiUP822FuG5f1uBavEjN15AhfMq1gzWs/Jcdc/zmqaELYfzP+nalDGVtWwub3fHOb+1/UKcV+utk94aAU2nfnzVIqwQWSvgL406PvPN5CjzxaJDCg0xpwkhHpgjMmAC8xymPxy44n6VaZLDpKNgQiUJBKN4/STwmEUlJftqEZsdL3CRURHCYucsoBs9OFfDRZNplJKQfPfWmXG3pgxr1ys4cNv+/kFMGiLj56qnt48FksyT1SjBgUQg6G3/thHO01AqKHDA7TEVdQOdCwYAPnGAM=", + "page_age": "1 week ago", + "title": "ai - npm", + "type": "web_search_result", + "url": "https://www.npmjs.com/package/ai" + }, + { + "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGYufEh4nD7p+ql5wRoMwXOXXp4nLBiAO2E6IjB8VY4Pd8kczLjSUcpep4BAG0CYBM1xLpcm0DTv5wNpdYWzTFw6hROK4rfceb0A3T0qixf3rJYRmxc0MAnOs/5zGbxOczA9Zb+iEFAp4LobVl/BWUywsjGF5tOlEfKgfz8nGEpwoqUd6wpJJ6HvkGGe6e98CzxK4x4XrdK7OV9mEjNwai9e31mcSBgvrqNYTRgYRKUsBr7Jbc+dD0AYX628zJu7VDdiojrcWBsSnfsemra2BU3KnmLzFKSd4KxWlee069INKT5RK2Z5GrzJRGYe78P6jt4YVG8bgyVNOGITGz39k3ktzPCph3lpT2v2pBqukefzuU952F1w2lTISV6p2K+tYXa1qhqge6yUITUwqevoL02OvoR+QDbB8A2dSrSWlUFyOyzpxAlOioUlPxNbfAmotc5LhyncSVQftFTX6UhNysipdBEIeYdeth/QWafT6BxHCyvzN82Tk1ov5Ixw4+S9QZycqy3fYjVXb4EfRALoxRNKb9O/IFT1ab3WrN2ROrjQ7rjdOeOHHO73ckRonDhrVb5OZjOjzTUoOSA9GaEry6v8/eEfo3makKr/jf1vy/3WnZKpp9zRFa6f1PU5/TIflX93TS7rSwyRDZs1IGWbSvds/CVvZKMTYirLZHFP3NoUkSCtFtooHA/7+kXNyeJfIJL3lOsyXEvKKCCcl3lhY0He5EdL4allQfRX9QGJW/cvKueiu+ooqgGoPpPgwTOKKVI+jDf0qv/oroOd3mxU1NDuLHGCU32+tH1LYGeP5oFlV1MwD08xNKd2KHdlzSwzDjK25H08fVsIPYgnKPxFB7X6F0+RLTIm05uRsQQX4iMZ/vnUvh21yY9Q/TyI5DuWKWS+2F3QUBc1ku3qgLJLR0c8wygGNmIOQ9eXBLx8pU8tzPguLesflxBMsdEcqx9cufYlVO7VRU8Hg9QpgMrbE78diwqv5fzHAc1WeTkdRbyI8kxiuGSbiR4Of7baYq+6EUE9IOobhwihTo588tIbPoJB3M8MoKWaiGL17ir7PHRlyemuxw2FsZx4/jl4Pfi7GsSJP4lZF2gAagXYVQrQ3oRk+HxQmVw24QQvRy045IfdcD4Yo4mBUl+o8GokSGs2QBlVpe4GGqW0e+vF1j7zRG56S0hT6oBKkjPLkzqMt0MceW/bM0qK+YpXs69WLWrGcU2ANome473/LaxtQl0GeA+neokQxWBy2sPcDvMWqhu4r7SC2+Morff85oylTXNaBdjLg2wz5RoMMXP3Ps/YJcxMB7IqSb6bXag8ToDiGZtMcx4pRNlzXhmN42ah0kEExUrHewCAICoVOQOZBIKlMg2ltTXfdaECXeQwFvqqhN0Kkh53ki4VaqunNAPlhdbOWn+/cYoIAU3muQZcOKN9REAp+kbLC40wEatbkU7rOxZOQ0UEVwKwgB0Qu1YUMLoUMwJcuOBIwHtszTyE6jiwU0VSXzcfOsmcbYBpTbR3oAUuqeEUJgvqepx06sSeLmZeUIP/z5yUvR2n4kXn/DMwOF1f0fAxmK1yd9N0s2BfRFH7cOSmyLBtw2EIoM+PuomzvdxkaADGs7ZOkp/fg7wnF2ZxkzThR1L2eREcYcV+QQyx3s9fRzR1SDAu5rPrjpF+p/iORBxhHwncXPRbTrFXdly6DgEKpnJsgiIbCaV/77CQcmXcPX79/+nGpPOwFY2yHdAzN0s1ECXw/6oUu02rHVv3mEOvLPGsA04GCVKvNO3R3e9LA+cVPuTaJTL2Lr9SWkt7F3JrIUF2bfO2Aizn/MpR9bt6gIuDTACfwOQZbi2Ht+xama3fzFcnC7a1FP12Cegp5Vl+kjBTCW6dC+EQ96IxOmSeNnGr9KAucXiclmw7TU+j7Xk7+dnV6MHcbtPvIYDND84ymJjYN1sgC24fZWh5z8e9s1tqy//hK+I5yQZ/K5OAbhwQCVFA3h25GnzFJe9XDMXkQ0kF8l254HxOY4HMhrAP/kNfmyKUM4KDa0xyRAgquazYc77BNLxOzLPCXJxdfrVc2yrGyA9Wu7uaJLMZlh6w9DcvHtsYwNXt0ii6VmVDeam2CxVec9GKMl3na4pdEdL+CFzle6usgCldABNvSKvlEKFdsHNE5ONEEQbQsOtFcN2nvKSpnF63BddCS2t0iJO4I/rFXtY+c3nHtZIBUIbvadfy6guSXo20woHHQL1cu5nKzUZjJhZq9pRwUV5FjY2qlQf4G6QDuWUODn0hdTBSOBT0mj7yHU1aKKnaYgj+f1wlU8b4xP4DDQIEugyJWKm2oBtb4OVRqpEOniTRlv6OMMkzL6fgLzns9OsEjBBOD2p7MgYoBFx0VTxz4v61Lipcu/njkkaTRPLufeXjxH8FfdrUWZtJwn18T5OsbUxwwJX2Pd8F+D41d5HtxtbNK1cEmb39lCf1JVv/WjDK7JknWMnbrc3wIutmJYyIeRXhWsV54UsZjIUCpkXWgUk41ZCO15HALE5CP520ZPL7lVpv/AnB5GX78c9WejKy9KyC6gMPrifnzDZLehE6dt35GvgHjcASTSKtxRowb4yxGF5bLtmZwSCtKGWY3Q5pwj1njmnry/bYT/7hMcFleV4yXvQ6jVJIiICLxoqeZMtPjDx7pfNJAd5X5yL/OGbSy3PKNDXtOwMNAxRe53/sNGRyy6CJ1LT0TjMBTcJG/RAk1g+biMOQ2vLzf3rUEVGjjRzEhAGkI+4w+a6q4+SelkY2nwLb6lgXBXlJ46KpvvxfdhXUzrRgb2fHgO8rBkfn4cudLV4v2HDoL66vV6meYgJO3d7S0AV6I779Af/itYol+9lc7EzmAU0wblnTMFgCEmHT+eIved7ASiUgjv8+2BUC/nK/Xsbui5Iyf2j3QSr+nWYQI67GJDWSJsZzUL096HtL+3lqpVccSJIs6i1rlyA3EBSSh18Pm5JUEsLbnJXW3THdUlDW8Xb7HKzlg7uPj0/ATTr+t8AeYL3awkYF7HixlkVepmgHZT9ZqLexQoqnIY7Mjcwqe4r9wcSBdVioLfBXVzBbNt83aeOQbdoPQi17/PgHWh/Rb9MAbLMtozHUdTp/azJzXC6p4p1zknBP4r//nbhPsJBDRwq8PXg2Re+XXOcBgghzmVkbWe6ko7Mzl16X7BSXHoE8r56for2YZi1qP3XVJS3ZAGhBfjbFI+5byq6/ASZnKnzHr8n3TDMJ04OA5TS2y4lscPaN3GBBuzHnpV54LHj6K/BRQpcByds+i+u+tzE9xOCbur0O/6QLPOyhDkGJCrXBjaAcUAKYMEvP+45qRZaeeptFWAQFUHSMKqcI9USXAC9GaU3EtK6FcUA1s/TvW1BjCFKdYoG3FrlLr4O/svbR65e9Uo/aaUKjYFW9XHCzID2JRgbDCSWfNbspud4djQceiebZQqJOWnYWbKOUbOGUAeV1ZSq1mCfU1Aiocrf/wJvnJ8/1/TwlbWvhowGyLDRZycipzZLCnUzHvIYQkwi+fbwHR32w5HnhIChbqcxMIeyS4gskypIvYOBtMe+nVEt4wIL/AXRXQfOrXxCoKg3kNMjfSGrnxGxgBlpr5iwVSvGypaEPG7M/KTleCugjnt0+gk+jYYFQKNJ/yo3A5bAYSc8Dk4cfd2n1mqcJlbzchpgKWkBZFdUeocy3n6gjbhrsutL6cnbXEw2R5kGeo/4zdkZ71g0slCFuwHiy+9XgiAzucyg98qAYyfsB4f5Q/5cOBqsXuDk1OOLJmOPagE16Zzb7O89utNHPN1tVeS7Lcm948oWP8B3ESEfMwCF2k1C8eYl3bu8lVLDZB9KonaPFDOkzmMsgTFAUK/0Z/NueGjBbeW/RZd9AKw7i0DFscG0H/k/Kp85DWfvh/IvL6Dco2cMGcq0VrYzkzOfAVPxWd8RP0fxTmJ/YGgyyq8SH0gI2LeT86GGsnP3Mg0koAIrDweFlpnILZkwi2ZorjQqteqFUyJZSavYcfluVLgejP91wEA3OF2spXYbNyH7rx0edYMVE/XCxKDBM8to7AVzfPWQYAw==", + "page_age": "1 week ago", + "title": "Google Cloud release notes | Google Cloud Documentation", + "type": "web_search_result", + "url": "https://docs.cloud.google.com/release-notes" + } + ], + "tool_use_id": "srvtoolu_019WjtfaZRafhhQGVub8SE5G", + "type": "web_search_tool_result" + }, + { + "citations": [ + { + "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDIpN60R7LeMwMmRxgxoM+5FjmBcWwA9B2zAXIjDdpRBukpgg0ai58JaRSF44Hc5lJsyROMhQ+fdYMYoXd0pMEnyWh1AEyw2uC3DlLIMqE7gQZiqxxXOuuCkVV1PnqaiwHE8YBA==", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result_location", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + } + ], + "text": "Vercel AI SDK released a patch update on May 13, 2026.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 256, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_choice": { + "name": "web_search", + "type": "tool" + }, + "tools": [ + { + "max_uses": 1, + "name": "web_search", + "type": "web_search_20250305" + } + ] + }, + "metrics": { + "completion_tokens": 75, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 16216, + "server_tool_use_web_fetch_requests": 0, + "server_tool_use_web_search_requests": 1, + "time_to_first_token": 0, + "tokens": 16291 + } + } + ], + "metadata": { + "operation": "server-tool-use", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-thinking-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with the number only.", + "role": "user" + } + ], + "output": { + "content": [ + { + "signature": "", + "thinking": "The user is asking for 2+2, which equals 4. They want only the number as a reply.", + "type": "thinking" + }, + { + "text": "4", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 2048, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "metrics": { + "completion_tokens": 36, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 49, + "time_to_first_token": 0, + "tokens": 85 + } + } + ], + "metadata": { + "operation": "stream-thinking", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "BETA", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 5, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 + } + } + ], + "metadata": { + "operation": "beta-create", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "beta-stream", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-tool-runner-operation", + "children": [ + { + "name": "anthropic.beta.messages.toolRunner", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 607, + "time_to_first_token": 0, + "tokens": 663 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [ + { + "name": "get_weather.lookup", + "children": [] + } + ], + "input": { + "location": "Paris, France" + }, + "output": "The weather in Paris, France is 18C and sunny.", + "metadata": { + "gen_ai.tool.name": "get_weather", + "provider": "anthropic" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01DgLUVPYJMvTXUdpVrJyUTE", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } + } + ], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + }, + "metrics": { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + } + ], + "metadata": { + "operation": "beta-tool-runner", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt new file mode 100644 index 000000000..7ab079bbc --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-tree.txt @@ -0,0 +1,775 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1415 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + ├── anthropic-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 589, + │ "time_to_first_token": 0, + │ "tokens": 645 + │ } + ├── anthropic-server-tool-use-operation + │ metadata: { + │ "operation": "server-tool-use", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "query": "AI SDK release 2026" + │ }, + │ "name": "web_search", + │ "type": "server_tool_use" + │ }, + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "content": [ + │ { + │ "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPeyh/w0GB57bzLO8xoMzb5iJ0f/kYhmTxsJIjCY/9gvu/eX3XWhg2YiBX86CXLEMor8pKwRIhRKfWQYhB1H1ysVE+Kdn84gvzbUY9cqzgjJNuI3vWYVhf5p6prt6WwG4hj0zVOTDx63JOlg3vDdvBnhlsH/7K3RHdQakLTP+jlLwROuVHWj6NHrow+9NxwVl5lFFUUFWsv+f+oSd08QI9CrxsCdX1zOdOHU4xgGtWiUaxPV0L+chRclHYIXxKTAEKCBFUZCdJNFq3J+Gh/qEy09DwTkMlYLnf1lnnchVXzSFcDloB3aO4NCOQEnkf39gSjgyT7KF+QlOitxPBKZV0f0vlglz5Qp3WgrqCGq5y2dzN1HnAc/vzTa2z9iy+sBCC47QlHLpI94eGBJlYRUcsnyEgHgJ2/jcudAUEfNtcgDoNuEP9hFsvaQs9Xa8DEzl6pR2g+HpzuRzuFAGFodjzudFFWoN6aG/HMvp+wuttY25heSNOgeZ8/c0Mcy4hG0aBLj9zZ/9dTbef99Ewfx/TGxZd+sAMihGXw2ZXHYlc9m2GhjCaQMkbBJfAVasMHmGpF0q+knR9sU7t8Nva2KTEF+IuBYaJikuj/xxL/a7OTMOR8zvn/JSVgRw6SgTviJycM4X+iNFVy2nrp+YpdxJAjWVXCJUsd6YUDKsIgUKVQfqkUNBFL0LhcE9eWNFmptyZIWC/pjvi73NVNwb2N06VGD5oa9Iac2PQ8XCnIENLv9IE3uFLR9bOYl0LR8iai81VPO3l5QHJJcdrS+vARTGrW4Ls8LYOy+ZLAHz7eFW2Qy63siqK9AivmMO3fJlMcFtYSIiATSdSJxqHXjimdi+qs9v0Nz66rmj2uX9Wisp1fH05YY4TcJ1GUzwojKZcaI/X8pfyIl5921qPZl6C/8BHt6nUWzdwqDS8WhqRr8x2ayFO4SDkMhjXaG/+8+ut7ETbJcMEfme32jmyT9tyuQrOp6amwwyoWNbQIkoDpIX394x+lo26u1wtMkjTRFOScs1bFaDXM8WQphUj4qBeMWhNaAwfr1klpArT327vQKZxr7FxfOuaMeswGPwObGGz34u6Vat+MwjIZhIm793O1HzqlLIRgoJkUyVA3d3/03vSjen+/Mxrjeu6mn+vXwH6SfF9Sh47WoLQbPRHQ5hOJQHoPbXsAcSGGi+q9HnChmcneC3Qq70PlaFA7/iavS+t274TE0FLf2BLnGuQyu6jYW4KHxIlx6N3m9nYMuwTgPI32DUoxFdUlvEMi2wARnJxjai1l0sofmSUp1gpIHZqw2coMsY0MfdONJxO34yIIT8GQaANZfemmcsFflOagPKnBQdY7uwsR0WDyH/asgn80SjTUCVUpgwZi2EmYqBPzq67UinGuePSMel2MgTfmPMkyN5ilZNsrou5qGVlv7wVSDsL5eZgtbZIJJpWKe1H1X51R155n/gn5OGJaE2o8TGsxM9IqUBs2Wtw1qhy6eBUxxoADrIczTiLkUg9UibRFKfqvHGJgF/uQQ4sKo8YZ+aprskETHLxgINxTrQ4GzB/SpFtF7TfcnhQvQO4+H/tVcGAM=", + │ "page_age": null, + │ "title": "Releases · vercel/ai", + │ "type": "web_search_result", + │ "url": "https://github.com/vercel/ai/releases" + │ }, + │ { + │ "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCFspfWt+JmUmC2I3xoMvPoLUtU0YD0FTFdOIjAr70jgxNqOtajClAfQMqiGoNM9zVbidTf044FsdHPkd4bPd6JjIsIytDLC5N9pQkQqkRnINNHuDrVsjmRXcm09BiCqyb4bUgf3uYUcf8wOxwNhOGPhswN0r5w9ghVXBy+22wWmUSKbw4OPJ/nf2DtPCv9/Qe/VSRKf1W/y2I3oifu8ULJd5ijLecm19kXW7zYeMgdye477SxiNCmZudz6XJh8GCCKOjDlEGjY2dCYRFfaasBl8lGLi3kNrCQxUy2oPGmUIJdPZrO+u7AlUE9ePyikJhUzGxUTOd2wxF88rIBcex9vDkQqMT4m8Pw5u06pH3ayuwY9qUm9yH/euvETlZdK6w2lSfFHIKn5aEnMqvzq7sIH0NQAd5McNiBedbDk5vpc/eD1093YG1Hne1bAJt3sccMakUC8wGkPZ3F3PSWBT7CRmyaWTuoEVRJRPB3xBvq1cJiWa8mCWfHSP3UHnmkKePRIt/YsZx1aFOed5Cz3Mo0xDpSAEdahL5ot2XDyKN0wpm2NpkKMsz+9fmg7RvJNYAtlZjDuaL6TmxUYgmFmyjYh62FrMijEzQA6AJesCNmAjpIiYKEwoHV6QKzRxPC3TYbityHuE2j0JHzHqk9q1nq1aKQiRyxwW0KmrjNF1ZTi7BfklEL1NUyM0el/PiQUPBGP2TEO/1NqymzxB60Nvj0GLDmIblsW8FWIHEvi7A6W6/n6F7T4E1tOZwCgqSqR02IAPmK4z0eyH0j0e/ejdzS+QxtNnXbopw/mFdiZh2W5L5GoeS6DvZp2HAstnxnq+5ye7/5sLso1cSCSv7rUJoPo5/U/StEiRxhOJSRFn8p6jGl+ZD41EhuhRy0CMEsxG4GUTnf7ZbGzeJvi1xndxaOL3bc25Ga59VlMe+Gdni38uoPH0kPy69KRJkFUl6JiwRHs0jiQvTvVHND/pLMoLOJY8GkmZFduB2PkzOwjNyX4Q8rRFJ6vgHtNvY4RURdgo43un74FfkNldGNOxcW6MVmsC97tDJU9Mg8TNYI5VB0iYEmxQ8rAIkJ5NLRs6B6JiHffFqiJEGXimw8VL8Tjav+SPVXKx2pZAF1+9ByG8+7ybG8Ekom0f9HH+uxgEVBKcClMWeC3V+ZL/U7GrwLffZbFVvYOTsbBF4WXliGIfLwsrSx49vXgXOQNtUlbHkF+yNN2B8IkiN78FxOPQeYidhh0QYqEyzol/NpA90F4EgbTlw3CDtcLNrYQaWTadEFPCroseKybQbeX+RLvtgN2u/gegLbqF6Ktwt6Gq0dPdL0o1UdoF6cH6CSymn39ZI2X4YDuNn1++Q/VROc0yEdMjG29cY8agoJgEV/64ctgUIPsJUF1dkhsL51At9iwrq7NJMnL/nVxkqg/UMbWRuAGAitnVxx26AKgNiP/Qw28DqYh/e47z65HNgKDZiJVY88N6Nvb3NqMFaPUbhS/Ch7G2W8q2hy84G9pxU1Do8J9DpPYr+HdX/RaBBTokJd/T1sCTYQm44n0z0KosXm7kv3ZeBYJYjFumrfQCkY/oW3tHtnOMEbrXH8tuPVSezYetEVScLJk+n4yvXCzS0wSQL0/Ei1H7aDzmN3f4ZbtC+vD4UlzAD3P7rLgBY5y5SCxWjNifsiAJJgWsHyXYdOfJQYjKnqVvpO3nGNO57VGKUTkNUdJGzD+QhqZeGuZsc8CDJSEFjpA1D6ENjhttq64UxoOUZF2CKLGv9jfMIms/NmAmv7KtsafuYd1eW3qoC4U1My/nbjpZBrqORlSIuWnDe/3IqEEc3lP8gK/9W/58rEiGsMBOh8UO/8bJCx6+UrlZxV1SWddPxmVqpVt0MbyrrglbYrp1jI/74GsSKrBhjbUYUkJPvLClne2OnTtJisuNG+5QpTH1aOh4d5CsAZrHtMJ9YBSw9dT1Sy6ZdiHm1J53+qG/wApdAnD0Vuyzbv5w0tlBlCB9xtOTeU4b8tKpmaeMiI+m52sYNJLAh6OENKpOxdbKpvhB8RiAnPdoeFamoUVa40YzcYvpioWu8je2irv91dJcH27CmJadDBxARfwxoEDsfJq0AL1d/pUfBx/27jVBVolCZ0mDaTWLKiMp+Xx0GW7SoQ+WtjyKB89xgO4qyD2A7t9K5H/5HhbpeTA29oyTnnkQyWHdVtBreUCOwwe6M4/XwF1Vwby6CnWQCLZZuBAT2XK8pIPS4g25oIHgNlEeo6doH5vauuodXyPR6hMebgf8PNskzrqNwyevkdaQ/zTGXy161a2Hj9fpWzSFW7vZ0y2WEkiCFFBdihQaXXgr/fiTGsEdPhmrwR13gL4sazVKI/WYONf6nxpDQPhSAjbGT72CWBY39OR07VXPeErf44BhL5OalmNMEbSLWm8r0r3DthXxoVvyur0Nxu5vQKBvQVAR1C+UuR/JagNZcT2heYNLZJVxp0CGMf6jVe6GOZis35qdaZFpkvAnMAWsAtwfiaZpahzU/h1X9tFZVihlzhHJoKYAjmwrLEMJTTMZptjB3+gA56+FA1r0hTIzoxsG1u48BzB5sT/aeg8GdCUHZM8ozMgc94LOu+1WI/iCin7KsNcA3BkdRwB1tZ9DfcdD5wJDha2E4yzETPeCgyEcGrcogfw71hak3cB3a33iyxIMWdD/BeLDxvWKA6uCYJdUsyyUIbHJ9fizErPxtZI0NxF0nt0Otq2Wkd1KpAOaSH58cnDnQM9BlkMHpddr5KDTSh/r0JNVentvMdnmzEThXYKzDDcFo2XZMa0R9S/c1jJDJrqVx8rzk4yaIV5ydOSWerbtm6ZJz0KUvsz3q5A95UC6i7pcGXaGbAZm48QJou8CS5UAGtTDqC2Qp7g6IVt+gZkla2F+rNZCd7gbBBkSHF7LaYQ0ZV5qhLkZ7oODArziUgIFVKdVn/bOuYKV21RjLuoS0nPKHkBB8gVl+8robA8efKv7u11NIzgD4OyWqhJZBO/tfBBtKvA5I9ehubsMBhmIyoFqHfOfpUDU5dvlmNFJqjDq8u7WUCUmf1OJIl7L7FiUL2T0+M8qOYVSaWhYa4xy04VmwY5q9CIKDvYks0pMqj6VCOoQzZpOIUJ8VyymCTQ10PtGj0V8tIsS1rTRo1swjThpUTx46MfQYJB0Wa5LWvp8jjn+pDQ3VODvolYtCoPty1Lll/oP1Gl/gJnWKzhPEsL44/IunxJI9k4I3kwzXCpUS26/PRNYEXXWMSkMNIbgkKDqa2Xp1UeSHoXNaUgDQB70uc3yvNhfKVqlUGvSb2NPNj4m0wfzLLiA8CFnIfLyVFndo6zzR4CXCIrGBJgXd3IbmIJ4kXr+QLxeQizqSsmTeI3c1lcYbRPm1btafQf8ra3sQpXD9Xau7GAICBxYfS8RyySYhpdDyNsWLyh5JQivD9iYRQA85rWiUNIf3R4dFnPrVY+I+qThjpbOFLQQRqrlI5SnoEWpnvaIFydPlHHwsHT1Jif5lVfG7rCAV9w8XrnjKhvtByZPwwp+QTzDABxwLfA6tpEqI5bFmtxsJyZJ6Dhug4yTw6VXIXjeTyyuGym9cCqWVrUyqvFnnzEPG9POLl5Jn0es90s3hLHf6JM0yRrdY5uMV/8pNaxcl5W/UIRmJLH8DLr9KtsR7EF19NHaBy5s1fWqNnJ2TaQ3FjvyB8v9PlPHxhPFeezjme+MfPcuB3egzhsgTo8/J5b0NzjbX/vkTX4riCURKbLT4+V/kAPdBH6Wi4C41M+fBSY6D1WtgwCc3sfL68Xxri8Lxq5SqoJXt9hwetr7kLmgcDRIi/vHVxQCfdhsQdw3IOxIAsPygtiVU7GT8WgjRnWA3/G3EWneNJ8a+5KfwN7zOgqn2pO+BvLQz8o5ZJ6Ve32XuU5FHf+BkRrTpXlpfnIgXfZ9qfWGHSj/Fmtw+/e8x3rRfe6+pkgrQd7rwXm2wbLb9Y7PwunftE219tuIMT92ABbJEcPw+yueQyNXYYhkW8icZbnfR3V5u4+Nl0aBOHO2rFbuhoa4+I2XncoFYXEYPdNm8lcAoA5AV/Gc6dV8kijN3gk+Vmk2TbLAP8AXyP/nxLKylVtyltM/p6wd4nxDjJRrCmQBBR/sq7gBoKpo+saSf4Xtv/HKNijUAjNYPbp98buOTCFOhJ9VIYcLDueBlPJTuxOkuLLpl6gNsOs4jNa5xGRectFUUbExHxyqWm/N0rbEIorGNFHz2LywD5KJFX/cYsQepy5POBzJ0i1H2NxDIb/HNZK1s4pnUgLGjo6r7tR/cy0O9H6MvfjTOqqslu5bI+yICnH8RUvV8rA90U7J4saXY+Md79uoYAEsJl9mSmk1HorSb8zrpOv/JaEjAJrapPDC3QcIQF7LKAuk3GM/a98Yu2jRZDBWsO7RGAM=", + │ "page_age": "2 days ago", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ }, + │ { + │ "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGqeizt50On98+WNdBoMrsFK9FVnQtBVEAurIjDFLw0WQmyfselv9Y5iYvLTMXJ5/E97vJi+28bY+3LiKoqdTj3dxo3DHvLH/01NRgMq0x46TJJ7syYvxhP3vu3rol9TFY5AF5RFxDssQrgdmSto8+X67nvWXlak86chMf+RDSriC0v/OLoGOoFfvcbRfi7iazvDmZSXG6XfVHoieREerkcF/d4yeKKdphiTWnjOld71JqIktcTOTC/SyQgtfnUyk81X2VLGQBX7lEXO8+urboSL+muUvuHedmRn4Jqk32C8WktvIqfJXdUgerRErO8dPQFqdCaWabjitcM10dJ63KWBBPFq6FdjZvnuBya79+x53YxeSJUrr7x/waRQQprcTT00rU+r4JwR+qDoUWi8Xvj+Sxo5AeBeNgJEOgdQJe/gwT74EdcGqd3ya3d8bHghBHAttrv5s3Tr3HjGfP5m7rMSomKbjHBh4IlSWcP88WRRDGnv/eOMBhzCViQ3+J5ofPsngcuhiAxFRCoGavRE7MAjf9XFBonxZbUAvSGj+G+9/nOOi+3TiVWA/OjNhhPFNTNQHW48QdrnR8Ty5K2Bt+KZny9EYbAnQDkBXgju3SIq7TnznUggl50IifdJqP6dGjzih6wJKoU6Qx6b7X3XR09hrETTGTbuVTMJtzAdcqib79YJUsRItrkgdeKTmwe6g1i7u8J+gyxAIYRFG3VbRqN/SQkQAcgu4c6rcp9+hiwkecSDjau+uuEoGA/w/hf7rkJO0KfkkIPuExI+wQjnLo+avekXfZpNQJpsa4VsSg5rwcMswRRRbnQGkpmShHXxYDBoQBV0NrMWCpdxvTZO9pyrJzhRLYlySQKQuoDCIAuSjCuou9SAuv2Vy0pTxIcC0toIg0Dd/5rrILxd0eu8Wze4L06SniWOUOlH8PLLmLqRoLPpMiatLBZX9H0OB//hHi0EbMHUq/L5TgVa52iR7/T2GKyptwEM1Ivavl5YOau83WBwy0BvpALRKuM6jEn88II6ayiA/GBg/ImuW/0aTZ19DBPCLkkm4RlzKyLUkpUzuBpfekvxPq/yG1zviK85pKtVax4lQqK1BIKudSOXvBnyfQRKubFprDDvxln89NSrqUP0ZMYbeCxFxxLOinPRZyRVOZoCoG1dFFANTy29DKI5L6KmvSd/KoWP+6jgyU2V/gpIQEvWodkgZUjWDBdxn3KAOPTOXPL2jZRLJb8LKlOeOr3KyDKa/nPsAVvKtLjOWKPPiF4/vL3BP3nJvphyLUTrOU8Bcvfl3z/qPZx4TtlRPW3AS2avnLuVM2Jq7aSThdx16vr47nc/zAgNHq/mFvCLLgQBS0SktpUKXPlL/E83qDOZEsNIEi4IvYYxBb+ml5ynhPwpMq48b3kPfcVB0/nIDvTJo//wZn5J7sKhBRrSAiORo5XggJp+VElaUDcunET2YgkHJpMZZlG7CDOmiI4jCoHPfRpE3oSMeXnzOuc25Uibh3kWkyvnP3+hcNXtmREIY5ai5A7vOEABirvusZThOruNuemvRjhsiRYLWOJ7q8rcaFiOn4kNpozwFRnaVrb4S4CIJs5ei+clLuOJdmRtFIImUkntZHBpr96YxMkZBjE/iwoelmI3Is1S0xpIPT8dVmRfyrgr0GbLqn7Ws33eJwGR8EssWjJyk5NiSh/TelT/rP17VzypmGrthU19R+7adbac16/DUZbAFpKQ0Yrj+cYoxZ74eyTfC5C4so1P7yvTuxMNmSr2R+cYurysTgECgOzH8gRx/ej7aeRxn5CEQKTeO1x8OKkc7B3ELTKOzHY2UxqPGFKAeM5nuYMJ7v8r0qq+6MX8g+MpWfmCTMhihLC+PSunQYTHzRy20LpC7Y6fkMlfPCUL2awB5wx2ZOzAJegRqANDd2ARZMfT4dvUPFjZbD3sU1nBsfh84Hn+WqOb1fA2DBufX8b3iX8+T+OgXk7z7ZKxO2PGz3ZjAORQzvLrsU31Z8DdJoi+Z120m/VXdHyO2yecPJqsJJrW0pZ2HgvwuBrUjGSudJK4z+UxKxlmGc1DwLkePPWfRLbQYItsPrB6iH2GuZAn7SipmgYOHqgtWqB4J6vR0UznXwDPHRedcnMwNAk+vAOjXzdOPfjduAwZODVaP3guRHr+4rMdwFOtq7kGuY5GO10cTmCg9d4Tc2SwYmta6Qn4tcoJUC7mQw26fV89gzktn8qh+WUlQ2St7Au/vSZvqjZz9trOKyekqn6/IJGhO6ea2TqaeNk0Q/piEMIVqEmUmihwgaVWOlPDN8WACOuubYrTwx6lWvHEQfq+aSNkVZhEHv9AzgOWs4yrp8BENTyyuQWYvZpH2PvSbNpeva2gZBVDXdxJfV8Smz+8Kfs4E/56alj0WV3M/GDee26YRqOO1j2VmK86Y1vj0GvX9WqIH93qhvFdjwhDFvv06KwVypO4MaaQ93qggqiwfhrpz2jDlW9KxpE+Jo0WeuuVANvo3WYsYtoZVqrE8SOyfto3bf5zd3zzwONhwuVmp+yldoNRJfPkAJeq3Y8SQHtevPB+t/M3poouruv3JZq9/7fUFZItriwawRoqbb8SIc9IugXXmwgZEJHt0z2EENHDLCvELW2m9Tiy3Oz9naWfrvmcdpnehL/wFneKQyy83moZ6JZb9dQqEKsIkUPhpSy+25nw04QYwT+MLth/L0cEI9kU0coSvWZNkOgQWrsRnnnfDMANOeYT2WeX6aREtnAwxYXK8EEx4L5Ds7xSPDT6ytL0+bjSgyU8sY5Aankmd9vfg5lKfAmRpiHWu3lWdaVNnHaJRBRMjIJTBWBjIwXvVHuaIJjEl1XXF4DlRVUN3YQIt4w9/gxhmX7nIbiVFCLT9QRAScSoY/bYkiq1VPnfJHCozdaUjKuOBMuYyX3Sn2LLXa67JSYL86//w/TvPWpffSihihKaNkNAgXfXMITTKjyaq1Ll5AdGGdGS53/sxPz0G7mUv1Egpe/r/bh1bh/M2SH3ZnlOvHQUehHPuLA+e7b+E7aTteSamUffdFxtghaoCzwHFp2Af9zmznILAgkw6cfwCvrtMmVNuH1QuXR+rwuZMGH0RXg3SQBoW0BZGQIqv4CYHLX7XJCOtTfA4MUIMNIv2jProhVER2IL7Bi79e2R0o5dmPtSxwfYewKHypGFr8j7S72WUL1S7zkvmnTDr5kHdBXsOCNXQk+Bjg/TAOzEgS13rFXbiDwAcfzRbH9fNNDQ23EVr3DyS/qoZHNKkQrbwJPF9dErwhs3hMMiHTSOkJOcWNlDCjil2e19SA7jts42UvrxJlq06iIr1wMUbM+hcGxN1JHPjuyFS/KjcuNyck6reUi8rraXaoe0Jt2pv2UXm2hX33J3Zc4cqOzEjZQu4cxFQbUAfDcGiSBJiJv8rWxtrvVRA7RleEAj7fs5CIlUf92dQjVnX8cB/eYzfj3tqPjAqdhFFX87kAuAGYOIBR2c3HtMkv6FKe97EXTOOyDDKxDVW/TL/xqRU+1++KUfLbobT0GM0OvOZ8o9gYleQy8dJ/WiT+chGCjKGWOTaoZodIzlb8OMUUF2EawzDmp2N6qcnToV7lFdN9cUz3RJt3gaIDYqGrdA2X7hXBpgura3OiQ6Za2sTAzqZ1o7qy/KABTJEACghkKXDRDjw8WmJnJQViaFM4+u6wfQYETddumrNnxNXPQ8jx3v+gt4pmvALcFBlmsPUJhx8PbULjdgXQ8q7hCgkp2UTM4wQnPBnobTtHMzOejyO1M+lS4WSTEETRqcLtYLt+OXYfXZpdG0OsRPUCHJweJ3dS9V6a0PpuClJAfH4+S8Cr7guDznVVryfg22e3BHuAhQOZ3VIidxeBTEP73EP7x4Hih/2A17i5bpzbrB71WvaQZ8DpX8yYoQ1NkTibzZJhzgUMGreKY4gdCfPLlfj/+in6qv3UnCCpJwuRXERleuUoZE0BY9U3jFO0dmRiYTiLk8ypGpIJ/frLJHV3QnxmftfomZvO/fMRYGWKMDZOGy638lgA7od3ZnbKiuzpJfZbcUN1D6OR3Gueyqdxi0uKkeZMrtEEzFbxdHjncAmKz9gV7a7L6Ln1Lk2w94DPyh11naHqB6K22eqyYtSwJBPfmwxdA79KYLxEooy/8Wk5JK25PDHECVQqJ3CLCpPLdoRv4YMYH/R8GWyDSYymC4zs7V/dMM7bcwelA04FU+tc266jQ6z9QmCJfKQYk4mw07z9rzsV2q+AXiNiwzgYS5zDmnd+ggjk8c1UBK1MfDXs6Mgunz8uObVEcfWKgrfQbn0S7NDnd+SCCySkQDNvMy79D0BrmdUNC6uMMMOq/dJTuZhKI6d9ZJAwDyqrgtxb4o0ZO2k4H8yderS4bxcs3Z7INhlU7DO9boWl1RwCD3a0oo+rYCMPKFpyfHDfFwWJfkFs2n4Ir26ZNRjwjVt5+ZM6QNBvq0L7HN7wxAt19Jpvezl6dMef08VzGNcru5wnJYnG1NyHY2KJ6q2gOfrk6Ca9GiJfRfwmsvZA6GxRXw8LmlA6oNLkhUMcVJ5KeMiMZDxcmoAbBytfQzFApzr/8o5oUoH0EeQNeq1O19Q3Q5RWLUepS5uIw4Zd0+NBFTKwVPWe261soYFP9zuC8NrX0ABD7K1MgIYNHXC+YbMhGAPF2IvBBcVCbKfiIdnypxfzp8bUx5n3O4OPFHGppmHCtbgIGTxm1El1rrSklwUX2byezD6vlSH/218YJt84WblycEwLTznMVmIRRI7xS2pY/Zy/V1tZDPGbQHtke/v6PT0OcAPfGESnXAiz/g9722qx4Bn8CqxThGL3V/1dAPLOJbMgLrfGIHU+GX3ysIW+V+kBIO/n4U/0OdFK0/Ux4S5de96SPRRdTSHriRh6iw8IHiyIKbhmM+0mxUrnPEmsdq81hAb9Vxh4aS1b/OGIhMbx+0SDqNshTPuftQ9k22Xt+foESADbYmNw7STJ2gt549U30XMobhrSSso1Us08WAU+suayA1hkMOvmsXfOQoJpvyXDfxtD2d1IiZFtjAVkn9QFMC720caguWLQZmd5/fhm/IkJh0ixB9QsLvvECJueJcBHUSfu7cX5DXcoR9d6O0SW4HBnGgkPPb5KfwtyZqB3A7nb8BD35YAX4sIOCBBdxDfNfRe/+JXfU346oXw6onR75Qvn5XBBVe8jbPqbkjmkd846u3Rnqa36+uJTUB2/XvdH7ByLOlp7JyuN4VbvPFMwFDQ+xSnKqYjFsCvR3SZlDktqpFZjMXUpmgG3Swxzj+UGWQN1HkWWBwP0bvMsWrDbThRmxzJDY7wSf+XSwLKQc9/xlPvLGuVGstUECy8ABbDhgD", + │ "page_age": null, + │ "title": "Release Notes | SAP Cloud SDK for AI", + │ "type": "web_search_result", + │ "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + │ }, + │ { + │ "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDN9ol0uSpJZ2+v5t8BoMq2HRp7ONJ8GgQ6+bIjAS2xnGjGTBkt+t+Xr+AhxgLDp/9pwvW53UFx7rNu0kgJdhrpG0KLbi6MyDCSA1QuoqyhT/5oaBOjOAbQfNT8ZrHI4j6lzvAHfrm/HyPJzXYuY/ZTHryyO1JjP2tuM2t9C8/fKnLLVko8fx4wdAwVgh+2YqPWJBU8lrUXRtRhB5QMCCtT6OSmKZSYPs+9LOiIoheIV4vo4vtiYhDNcNlZNrYMYAB/KHgGdeWq7w8qdhziiPSBR67wsn6HbPajJ45OD1Aqd5ly1VCet5eh4uZPpXu2dwapPHUs2xRTGd6GRnkpe9VXjRtbKEmKhab4iP0FMU940MpVFTV5adJtUO0wsce1+YxTmHiRwf4qbWhanQzCm/cd/1Upk5eCu5K2EH+xY53HC0ADFyxfkJVMBowvYDngLMolSNelRqsTQLvXWoNBR7qkmH1HWwAuAy1ktIkARPcVgZXixs4elkmrgipJZxaq8FwyYmjNZaFUnfRh8+0ExUNNKGk5bQPmGXppHkc/3I7BH68j/ranbUUrXxw/s5acAacu0ZxRGFOx+0DS1nylIyfiAaW1yrbPs+tn0w9hSrUEd4UiiffYLEZyjcWHXL4HQ13KjEHO54gSqUV0dy3fOs05ld9K11Sb5B6ve06fBwxgWGE7bSTm0asZNc0AzlpgA3qELRdzt3nCTRp5tzW41FnOhfRt1n+xw6dm6OZ88RSrTia8ICz9FxjsJmLl6Zcug+7qTaR2f/sK/eMNqoz97vqaj3oMCiigSYGRbQ9VTGVgBQBRmPXGzdZIvjM1B/CAFjKKTbH3175B6fdZZp6tiBM5APB5bSh6OQ28mXBoIHvyUfdVj9FHOLhp/KnmU3sAbkjvRGLQh/YrgAmfMRjBM2OrbaVyL/s0DovBcHtaw+GcrsN8keDPd5gk0qgj2B4i163NSlMPMgbuV2DhRIInc/Z0IuSm03TuvLvDkjv0X0MUbkb5vK0pAAV6zhnTTWsSp2iHxEuzpfUsC7XWTPb5HB36fyle79OXky2AL+C5amspqXyMnxgx1Yd92b7kHP49UuhAswSOYGPdITIwBfHu1b/103OWsHLo1sU5bpKGe3wJX3XuMSW4Km5cGqX1OVBqFsU5rCeBxmULAE5o2f3bxSGVJPYEoAsw1uYM0oDcfDLf/6eEFauRu8gI3X/FKn4nL611I1IqXUyB24BSGyGOZ3qULYqvB5mKED+j4opApnc7IudqvwjN+6epDk79wA+oRsTfQv2GEvG08ncYp3gx6fdWNJjSMQOTAG2GOh0m6XBt5kJY9iRXw6UZWq7u742MdqCOHPvcCYuV2Z3Q4IYL9r/mSWqIDtruvcBVvmUJBOqZ5lex6F4Hsj6Bg8/7uu0+O5ez0Fq2epYG0zB/S2QedGgPRUNJvSBG3lKrmakRQm9dulcx01kx6Mbr0rHL6OZUyBglk0EhFbgOLnMmgob4m+PP/Lhs9Vg/qByLpEKn8e4QXbiUCWBR743VGhaVwV4lZJQH3xflE4hQVOi/KwZ65rj8/vhW7kiMSDx69VCZbG1ZqLj/oBK+UAwuWis5seInKcrk4ZOLhLXMirN2HnqnJZeMm5rt3aXPY5pmer1BQ8REz0yGVgGbRXOCQ+ueQoUxZ14zy5Z6pOqdvI/I7VK/fsX2/bbN7uB5VMd5qNA7djF8K+xj/MDBcB9Sf8Z71HnlgUXHxQhOrVXRNvgjlgOP8LwrfUHCjDxub+CZwwB+iaxVPUXHu7tWZQM9DpRleg4XiNaQ7OAWK133eBI0HbUO2a1auBA6K6GDc9slUB8OODMdTIW5+GGhh2DnQeh2QHidHp9PdJidkLzPCd/KMd175zric4hOPtIeVokHkbU03totGTqqmFjAk9G7b5Dhgd6jDDY1VynbadmI86WAJ/yZNNirX0yP8nSmglb2mNRJF3WUxaR3P+tLYUJRAw4KNupcLERta0wOXBoRCmSFth/jG/b7teMAb4kewXX6RFIr5GXR+2dJDBDI8dC+6Jn/9NXSQgSKP6KC8zvweyN7ow+WTuhk0mZYMxrEGdrpBNyxZsrZS4kTF5tv+zHMIXTje7JSLqdMUNVfzzOchrOAiJo5C/9K2T4y8lX+YlkSDIhK1IB46gUCwc0HK/CsFx2gVThln0zwAeU2TxQtCf3f+fnuuDVEKXXOM9re+tLmPf1ct7h4hbTDIwTDNpj9fR/YbKSV2mGxPc57CbHDabHBWwhGuN3x9O+upw1wEw+Gn6QIc5rQ3TFXvlDCROXEqTbPbWh0+wxGzwH/IDZjxUwMM7ouO9YhBtW/4pjYAPdmp5E483bFH5SWODPNOHWXfPJe/I0NveMSzFQYLIt58Zz4EVzhUsqwsjfFxB7SjeWHq2xLxZ5D1uhimXjTVKgWq9zjHLvhn+EprRMe9rTOW5uhTaj7DQBJrngietkEoHaBz7yc7xoJDDe20mY+1ewsytPSvuiaHzc5JtAVW8jY4hdRi1W/4Z7aOMtpbZZKtjo8YP7l/+2q3nlULaZkv0ERN+xHL6XDBJFOxtu8GtXA/2Zm/OUwlOhNoetdos31coZlg0Bd8gJp1B6i5UwLqGOnKY4FjdNDNEQiQwxt9S/fKvVAcxtxF0mJjdYLta7jYIgzUWfqQ+LGr7RZhWxXfkyUqOV1I06Wfu3Xv+ToTKtvoOuU2dDUX3e2kQHOxvh7lTHoB3T3sVgjF++5OqFRU38Z4950ucQs14rTWrw4/cprfdJfYU4ddxErkbsXs7dYKab/N/tpnaHfkeNDW71kb7L/6H30T+IKg1HYXmy+EdPpHelDVxkUncOpYUHoLwJXnaIzxmtKj+IOZ3KwnSwMDRyFaOfLN2mRxluvVB3LVbQkRUm8Yk9a0gaakVkOuusYCxc3eT79FpwQ+/CPTab4diuLal6RMKgGEIXJqGMRXUyf/Ul5l45mJnbRciYYoisnBTdqkjoZO/voyJd/ItQYCJi6FRCS6h3qksjkRKuWl5Lx3Uh4F2YDix+NIDy8WCmrasQvuyuy4gULkfgoIn8ktn7DgDRsH3F5l33CcrggXHbvzR8np6mejmuQtTgaqfC9WTHEQqDGjVN5Eb8cFUeV197x9498STwxe+55ETchFnNVzHcRPJ7B//1LLT+MRdcjhbyyl97xpUiOYffEQ+MnGhqL6ueyC2jpWVpWDPtRFaF95Zc8aHPw4nTDJrzPdq/VKmcgsgHfhszhsJDLDJ0xKIIDL6t50BFNTqINjpGTt5LUB2ImzEqtVTPxXVf19zUojRXuUBI7ZnSXjchYiL3S+3ONwq/lVW9ac4X8hMy4kHFPAJO5WgBmhiiBEoaPNypB7I07n4v8GiGVJNGGbG/RtDBbpDFc2tftWXVumqY6kx1L7gpUr+BGZspGkgPkXWkf5H0Q1AF57EYPbs4OnY7uQa0GdGkZf4n+C6MFMuFj7RpKMJdwuObPLKpHxWDFOClbyeydgdDJ2/NvTLqQRdP/Wy+stnIXNMEeALVNfPqjKxoSQZRCjq7wv8zMGsiome3eOzTmxd7C3fCfWV9For/e8M1Bcw0igMsz/ON2FYxjuuv0p1NfUUurqgz8G9Vdh6yv6FwfQYAw==", + │ "page_age": "1 month ago", + │ "title": "The next evolution of the Agents SDK | OpenAI", + │ "type": "web_search_result", + │ "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + │ }, + │ { + │ "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDD6D8gCUj0z53z/jmxoMvvotV5Cja2ew87/5IjDCIH1vhRznhHZiGL1yShZJLqLK7JpQLI5k8NWj2/7DnVf96p2SRcz3oSJXuD7arfcq6x0Oxsn02uS3i6yrQVmrSI+0EBsN24btcHfG2Sgn8rs0TLvYfS6mE2HH6+HfmZl6FaI87BRqq1aWGVMqHzoczMXDa/98BpVhDyggIDcUIlvkUln2tBKpi/6a4h/xliDVJXofPDeAoac143BkUdXkiOddbHwvqU2CEGBSYtnTES0kUNS+n5WNWd4DkrU7kQk+OpZ2DuCvmX7k5EaoSv8N7TGg31ZfsLcBFTw4NwHvk5KmYQnGTKfKLrCxy+G4Z+2+sWrBRP1THQ/JscDNBowDxneYNMflgMlcR2/kWEUB0fcFk7j9lxjaYYVsy4aD+3X/tC9cEiWbEQsDjdPsP+ndBFOKjXdBXFXiAga+wr0nXXUp79jFu8YpxJSM6tQ9YacCXD5u8UE8ZPZxn18+7sYdaJi89Jou6rcta6Ltsf8yIcJLy3EFGsz9aC1mMX+JCc3hlCIQjo4tsoPVQHYVon7llq3mN7Wie9v5JWRA3ce0nXYQ79Tp2uEVbkKqeh3krTqkZbUmvsgHI9TYdi4Hx9bMG5uhhFvHoSqzSgPTIjNfTwBWzocvrYfOGF7acTLrQo9NNPnbfzTJ//wvx12F38eXkt6dkMxRTy/RoELFqAUMC2yimdiZX2HQuLrmgoqb5BOTvcRzsIzSAh8hM2WmO+9DMAhET3jgdVxahZi6u3LkDY49OyNqIptrnMurrncfniemzSJcrTcno+tmlLb6y/kQs4PQM32u7mH6HFtN3yBgxn0uVmmnVbHKnYYiOZvz1IDsLLhFwIV4gVNcLj0AXpOvTC3C4Ya82X8ya/+lEPK9qdtf3zA4Nuha1/LMQsACb+dor534p2IKb4xHIQIKJ5G/Hd5dg8d4W4snZkvFctOrffjMtKTnIbHJ/mYR8GSPHrx8CR4RVImv3xHI/qltcHlijGnkEO6TA7IMbJprpdrCiL8iiCiGlDncSZbV8ClLlqVd8fZwPrgXKRp+3NdRtbKh5GlMT07EJ1mHy0CaZMAFly8fvH5X64CVhU9OTxnf2ML9lh8y5LGQZdj0UR7YKfS75XlClZBWHSl9v9hz584Lwi3arvPY7QMYoXwvXjZrqK5vG2YYVgbmcwR4jLpV+Pocj/srCX+6AD0r+G2/xU7fxFrd/kASTJZvAkGCoXcMwpOrGPRAb/Hl8HxTtghRhYpkDm1EYKQfzmbYkWqarIzBjLvWfvut1HOioh/rzuaXACcySWG6KwokmLy+LJDgNYBLr+Ksa+JV56T1BThazvp5JnvHKwf9vtq5j508T6cHndQljcgl2NVT1q4QEg3azvTtT4QNbBd5MmH8Bv7Qdrm38MoZObPyEGyuzs641Md0q6EQuBJD/JFmw5j2GhzqbDMAeCC6YM9iJuoR+LdWIDOQfLqKTtlxH1iEYxtPiYHEtwgA625cTxUhJ131xLbgZyI2Kal8i3qGehHCatU36b7xVHsfHDsejW62eg5PFVGen2qDLPz2O63wN03sD3wE2YkCFUgL+xgybW3o/QLtoxXGi+/QK3ow+CsIM4rdxJZ8A5/sOnftHmrCkzB1HvA3QT/fPYqjU71GIWzzuBRajDZUxyRDA1dWg7AGHr+6espdMmCp/5blwau5uXLBsBTskhTJcRXTbhISV9v9+EMniix+Zrs469cGTEv3mTHlVT5Xf9GXEWzbkrIWSQCc3EZIqddla4HMX2cTKKHKRRXnM0hAIH/CqSWx79bQIAwKJnl8AmIPGR3Nop+pCC2MPjjcfVszqQ09IDPlht114JCXXHj16S/LKGpo9Xp8fpmoO9+U/QprLf80OecdJdmANTkK7VeCeEP4Nnt46JhGL4eHBPJz030BvlzGHDMUwV1EkQO7l0qdoL8LFn/aSvYmweXEp3zgU74Ta3aKbr+WJEfYfM/Nia+GrfGxjkA+cFM0Pw7GmRnNn3En9yy67nnL79CoGbwkzF7MkPS4dY+SYKQVSlhP4xvFtkk+kp7Qu63yIp9QkbaSIvblwwu03SIrBedl3D2ORoGnZ8+UXglDN7sYnNF+vm0HUwYHb1gRuh3NhRzt95ZXOdbP/nmvN5cy1jfySmddYTjrs6Krt0e4gNXQGYNHAAOh119oCRg1VV1RwVvSHfB3EHhSQpjERIFcj98tZfK/TnAdDapCDZklBnS9yHh3kAfcCqXxZxSKwasNwXyDY74TzSo95a+nmFWkOlWOkVLMEA31QWuXg1uQiV0kZBv68ihaK34JrlfkqWylUGGTExU/5Fnv87g+osHtGsSU/qFRjhtCOTKv36a4bZvvodq0Q5VbxcUWk1y2iRiymgCRsq+JuJOUbz4zFBVpk1MewBUpMqD1fmh3cPL+/8gx7OpmOdpDjF3mnRetPjIWhemPJoXxPepa7t3i4Kj5CanfN5GZVwkvSwKabwBUPj8xQNEZjsGvMlU2lu+zE7QljtE6vH+3zeZDPJUjQMYW7mxpiku9fk1854330BzW9y4+SEJwHHrWWJhasDC7qtqRB6NrA3NBJmJOf2OH7RezjhW7HZ4gNZ4lAiIh+3nj0MOdvunOInzxAO8J18SpE5OaG4cKebWbd0tSBOEvInZJiiXCUybid4fT3y3EH8jzNYM8adqo2eMFp3SJlzSOLhDvXjEGrHZSSYAaKo0XRZsKc/PlYMe1KWvZaq5sYlAC3LkUwAwYv7k5/hkK7sQVCeS6+Qz5ts2VPp92RsEWx8kI3lxdiTArOFYU3weGgE0/O5yG3+HyMAmxRi2FA0YLzHLi6aH668K1VK82za9teJoXK+WKaZQUgWCNp2r1Ss4xSzzdKPC5Vri/aZsxW58Ax3InWwYE0Jh3aF14TvuvbG0JlB5cUSrl0648tMROO/LjKtdqAm/SmIz7X3OrL+gIsI4Hko5M2WuwifRMW+IQTDefMDMI8s1hcO1ZgIpGODzgQyTQkBY2TJ7cQIB+CrGSF6uwZ+a4bbo/QC1r6O5VuTE+FsE/S2Ph+0AMWtAJp1NrD9QsgQfduPCFPMZTYGSs+OsY/qLKI8qQ9HueQNlYNWbi1ynAAp0D1EzIXu3vjmFTYnA8IYLln/PDJS+66Zac+TtD79spj06iI6oPWMG/GFmEVeetTRHltEuLAO8bP5D5A8j8u5GMszAqQdtWb3/MQNIIeZkIhVxhW6WMaw0oL9JktjWrH84ynBr5DogETGmkQhhcWay3KKfheRrZP3kRu1tPs4TIdVhuHLx+3XjxiQfk+7VEVSwRk2EDt29DyN2/JBxlGDMnzy5KhO4Mp6WrHks1kyvRFXsoQuWDm2S+nNm0j9OL3cOmQy5f9QQm8RUoK5Xk2pv49HH2qraeWfD2m1a2gOCVGOw2FPzgSsg6jSPp3vzC13O64la4hdZ1KlAX5vKP9EeTJeYIY2Xn7Vi87UgVMtrTAB9Rb2OgmvDYI7xPmI7eFfbtxpdlDhOvYnYc6FzuilHPm0uQ6cu/9b7xl90K3bZQUSHzeEgDNbeCFzMMur29XcN/A2G3wjEKby86Mkfbbhj/BzdD0RvYuHuWJFOO8dapxulPgjm6FJURcnONSHUS/+p3gTWKy5HFJitzyvpfL995rm5hRRNHtkl3b4I4sx6tfr6Fnz8p2rEGonIPgYvQT++O+Ry2xKCLv5tIF5bShdG6zsrhf4+JkO2g0UXvYGDIZd0Q5xWDivV6m28pKd+bDZ2eYFTLk2SGNHPJBdZ2dz1RDOjwuJQvx+ayHaI0PZnc+ued663wScXo0j8dyEfbQ3u94LqMMRNuHP1twfzBQF42TTxRUiy57q+xa6D7BXosoSdHgRhGvCyu6xYGCxVJq2IzXfpqOn9wC6D60uaMq3FPoxpZFgcFkxZnhdHC1rCq+QWA6KG+/kTZ1H1Ol8fd64/7J1UdiBSB+HNZBAj+Gvu91N578mhK75Ipkiy/hsGpXSpuOFgcsR2ND1LfpB96JUJ5DZPhzxp6eENjn395037Sr2nwtfo/FhYkOKQDoG31HC7mB2kU4l9eyiaxJy37l+LZzn8BpuO1kwThbj/NOPNfm6EUSmCOxAcQI3j9oJApMkXjdU66dpBSJ9mr9JR1eXS7JIvluQXT2ZRbauoYPGi/vyyhxQ0kPV5liu7d31DUBD18NMK7mFpbtW1IPFSdQ8Zp/iYlfP1l1tfdC9qKRS950P+p/tEuSFQDZgWNh4hA1UOwSEFUxuBBiXVJSyGDv8FwdBQ8kCRpvjUMtuAQi8LoeriS+QpH9vJbCh55a9NAJ7H1pHhfhNStztjUNKlutX6WlL8zGnMP32TMQz69VNDmuJFK87PkrqiXTzkjSTO4K2zlhnlDw+hlthi+plNzJTZoL67bjVX5wlXs1ulUt73qa+8ephpnABXopPAl9KwdNjdSYTyWwIk7gU6HpQszOGMbD+I6HGCila1Iw1/9eW4mWPu41sUOMqUGxJwb6rWZxnIsNufK62/KMZNHgEURfjXNvXVupMTdSw/+y3Rj5XFwhOhT5MhbwSUAsAFsLMi+CZoA+0ewIOCx/rGHlxta0D+IXUZQ8sCXGotam6HO3plT7WH9u8CjF/J1h54474ajd+WtPH7OpeElaxrgIjxTqsSw2IfUR3LJZNLY7YQocwu2aIXtQf3jxAxycCYYct7v1Yi1G3Rind56JqAHE9fOPEh+NjPmX1jS+KYyENXaefHdDO7V4f1TDGnb/MtkHnVQeHF9tGW4UDPm98DbfUQoJOZJ7+psdTVqocCAwyldU+LYFSyZ/A6Sb5PPtj/kzwdRl9hXwV2Li0SpkffmaLbk1Iwe5iC6pusq2eqXWSWgCJbnAZiBSP6Vv4B6XqwV87ei9XV4ePmjWl3daoqsLQqhTXzQPFaZtSA7/v6L3Ort/O5IU4OdFjPP5IVO6gbKlvRyY62VSpJIxEfi76sEjXzNk+gv4VEZ8jRfvtFNwfjGxVxJZ0X2PfjtJKLiNNA74ieuCkVUkW2HhhM9KTeC0UqQCAbP31mh8omOSPW5DLpotSPsvduLZRJee/twd1AeJ7uGYjqX+iwB21bDwagLfC1X+/BoPcwlEn3BoCyaB9S1JGDyP7ahyFytmhWpgxEUdtbkLl8o3Cxy/ebbliGysr4v5OUjAlThZwhOcjoPZ9nRuJAYAw==", + │ "page_age": "April 8, 2026", + │ "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + │ "type": "web_search_result", + │ "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + │ }, + │ { + │ "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDHp7NHAW60Zr6H8nyhoM9qwhyx9pbTyulCLyIjCSxnQLBgwDFHz9Tmwddm7CEGWVaTnpl2JXFhnk9GxHCzZBP4Yi19O6FghRVAv8sNwqhxG/+swA2faHCZ7s7frNn+z8kSdq9hyXDhNeTLLr1eCu0bruQxOuM+tht3ZFYdb5hj8/tOIswNxRbLlT92lPeg6wRw8XzUWhMaEb5aY/abhDcIvRo7JOog9TWzvf78al9WTmRua4eiLgH9jPfNFkY5LoNxe5fW86L/B4M+vgwV1jMdkfdrj35vp5CvUh9VC/rqJb4W2zZ5DalvR3wN/EjPUmmKOa8G2ZpsZ5FVFKHhl6SOEP71IdxqTolZXrk6ECtPP4612dxP0qLLDTbEnJqw8/Gcjrb6UrpxZ2d3lZu+1b4XV9FVtFivJ1T9A+2yjqs+vA/kdFaQGZMkxhW0+gbhDfQyQ1RrAtb5eOu207NaZJOhw8HM98X4MMnZkVllm5W1zT359pV7zV9aXTsowkcp5aleeFfA25ChIaVW3TRhQB+ra+aSk/sm5k74MfNoYw/SGzuHUFlfyeUoRNGG5Wf+gJvoOlBxGnZcLcKRyBUGi9tKIP9dvt6AXrhKyFUr9qMBfVMS0VSBZmKKtzFkeF04KmX8g0kHBZDm/+CNEIVl7K5s0DzhQ+aI6PgpkAG7mGAGUp1tg95x1Hr+JQgKnKY1TuoWWv0kMoD2O4aOMpxBFyvMPBaLEsTCpnkORwNi2EJ5HbgYSbw/6BrWpLjF6c7M4/gtsX0iFsVy+mYjTVw52cc96BDbkXdo6sYdqYSOZLR+P+XCMnK6dN/r4r51OkWZaYAhnrWGMs72EfYmYKIyRrAMJPsjlWXmzCIma2tZTKinM2Zay5EswQbF5tmEwW5iMUv/SkbHeY7JuthAXMIDf/YahazZhfL5+91en+HMFFGC7BKKH/FZP5lxOUNYmqVgI8JmRcOekPy4U0PvY15zIiYNqQwXj0q69ug/fDT9MFm+74oJrIUj/JOY8+Ye2nJlseIdcQ4wRPw2GLBovTuM3KlJy05MaCjrSSAK+zNRS6pSBWGkVEtLfLwyzOMhHlb5t0gsGQj+C7FAaqdADDAIwJAUUhCtWizFtOEB/nHozZCKjZVr508IEbPN/XDE1Cd6urTgGfp6awEHv2Urs9cx9p5kFiy5lWBbXy2OlxG/J0hN5N5+pv6Ta6L+tsmRhp03xKkPWMqJQlgM7ZNNDS2d6W7uRxE/cQGJ9Kn6YTqIToH9i+3K6DB+8r7Awxl0S49QEOZVgajHu21LOJyOzNcLwO0qyLlCq+VTj+q6hkQPhSGF365WvU77iUpONEwvSv1paudrVrTxCxVkg2qWSOriJeyoGxr2CWZTmcCVG2SKcJCFMitMrbZY0UsMPQBh5aRWiiJY5sDEUjCW4MwpmGqBeQQzJvkNgXuT9BhTigjNNVaQSx4w11JGbcYQfFmVLvslQ/uPklViGznUs08RmUS5djSuIdSXqKjc4x/YDBza/Cb9u3Lz8cv6FKnvxAfO4vPzw/k+5ie2QWCl0cwKJDNgXBId9178yxvgzPSDUp6tQw9OPTNnZlf+fKquAuofPOyRs/IlY44OOrOCh32pfDYQpWFmfKnm1N55C95osZjF4b1J53+dkJpC3HNr6W4vjSTqje6LoK9Qgud/SBsRINk1/45YRPOtWpegZgWfLTPtK1ki3W+U6D24a1H76O5ok/TUZIjSl9rDwpsqTQPkO7IoOXkQX5aozDGMRVhZvT6fr3j6pb8XGH/NZKLWd7Skxu8zMMxZRdeZpLH7XoqAYbeR/hnG1jSs6OUshFYh256vGD69eMfPUvoLfwoVcgqyMQDOHlyk9wmqtBKelsEB7j6h6jkK7rFzxMF75KzU30ObmY3IF+aeom1suL/JpuAuao39WOSzJUeUc6TuQQSi1wL+RD6kwU4ECnzEjeeJ243fijckmsMxJz/FK4PQwbuffEsx27kt6wggQuQwfGHi+G4L/FE4cRWZV6darBkFNnMHvqE42QUaEwxLu5h+MuXhoaIys2MmakQj2Rg3eSPM58XvLspSCVE632nWXKfHc4fOqzTIQ/kOVy8v2X0Hcpek6y0AXktakRTjFP8KhjVf2c91/2QfXvcioG1xx51TH3grDy6noKadu0uFOitxR288Xs4s79HKd9AY0dlqqOb+LdnrMfbBIi0ampKVGnu16RRcjJ00w7D5p6XfdRWHyO7AP9Vf4YM7WzIiub0Vtp3Z+MFB2GIZeFPWeU0RdQJH0MRBG12Psnp1Vn5vjLQGt9l2kXsEWTm+EIsRl2BvYKKCUkFzCjmm94nyPddvay+77mPXOuviB36A874vlM0b/0KpcPVEKTdZRwNFVVWDmTiqta77Cey3M9UZq1KLHsxHk7xqiuKlhze8SOVilXNK8q1oy1z8F+dbWgbuCdq0Mz9kPvCUqwATNjNN7eCFKjjcNxAva67ZsporCi9YsxUwZBC1L4wZiGAwK4NopPCkQmjJaks1fjWiVLcrXqIe54M4HvOgcykIgbT/xMHo20waOllfJUNImMNgFLDx1tKZjRQ0xBCedwnjWsrJEHn+7fH3aXU04vP8DROwLpE3ZRwYjvF9i3uuTLL8fo5RTtO+RaOctKjruNi0xEDISh1nTjpBZj/qmyASbUD4DOfqWhUeNrJ9Qo/vN/v+oTwQB9kCVyVxr58U5uLEMj/s2kxmx2sa4hmuhen7h6CV8H+e+u+BcXUpZ8HXhde1vAv6VNp47PCdCx76ovIdsfihdJByXKhJDnsP5XwCnvIy95+0RzVzUJcSAyLnbl0VKPwqZExVDVbN2OVlDqrhAyCasD3A249rqTyb6n1Ng93VQyT1jJ034kJ2XlEqvUvIsNGL1jDZ66aUEBQZuKWZMVBCKdDEPdK0zT3ZGKl80CVdbH9RYMjXs+KbPk/4nPsRK59x4Sbzacs7e2UwEaHf79UizxkPzk/gfyCkbeD+qK0f4hbUj0asfewKnhPjdxyJ38rAO/0hgD", + │ "page_age": "4 days ago", + │ "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + │ "type": "web_search_result", + │ "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + │ }, + │ { + │ "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDDXQvvZiaQx5GuVS8BoMuqycwt0jjZsQPxDzIjBI724GTF88TbzUz0t+31OKIzUXDo87rnUezvuct27PzRnGXk4sGY6bobiUrjrIzwkqjSClJ8gbHpjMg0E9usP6I1c45+WTdRESXalxCpRdcDDy8O9xU6O+gatvKOKg9lf64DKRP7oOgB6C606hPDHlLXtM/fOrg/5cX3dUx0lXvZ9FzbnhAz+gC+pE02JR1LgHAMyeGY0wBEAiuSaLH7dFMT4/OmxzG0kD7UB763T3bJf5iImGudUzz8EjqS8jDX0K7z3eG0E7/ifIDPW+Jv0mm/Bh4hbdB7DNHD4CRnw/Yc7LAlfcmem9+d14g/BVYzvKk2zUiNkuaiDc4SyL6RJ9CxrqKOy52ym/frIdqYV07vEekPFZbqWtF+YG6I6YncpibHauarxjBga9WluAC28pAKCFqdXV2XwnqPORsczWu3Nv/HdboSKRRll5NE9gJDCqbr7MRbPQJDAXLMI8NEqENHV+mSH4uIHrE2HHuwA5w2uXOLl8CGxYMRc0xmCo9ExEdteG/f6Cz5vEcrjC4K4MIIIQC3UxNdiJsRGXSCELH2HkfF/enMYcpshD2vi9UqTCfCZB2CE1O3zIKRWZ9ACTErGCaShnH8d35SXzCjB8cNMIfPuURqbJXuCO8lBb6UGDgxlsID/9D7bk2Nxw3/yW517jYs0m5cC0xV9p7Q+GzqJwi8G2sKH6VXCd7yn+bwlRKHBuBpVUAono4pnN1dccMxJ4Mr9kOsjhqTjPNYp/nn9cudQzGu+YWP2/z9bqR4/4ytSVzViIsR/abOo2YgCTnjar30OEt5vC9K++6YFFQkHUV5P2I+zVAQBUOQDxYDp/WR5EKa29hkE7OU/PzdbHYlsQVcbT21lISs2uIx+MolI4c8Dw8W5j+Ngcea3MB/Ozx+t9Zsk/Of0cVt1fZwy5kTyNNTyl7zjzarczyAD1w40VPxsX0yf+iD927xUfTECcor7GcA0LYSgMiVUQSlu5UBrduWZk67v+MqQb99RKTpVwGDUlQpP3sgvz5ivP/ASfGVUMsIwy+dA0Trrw/itf//c3/fHr+KGuIcFOLekC9VYJOjrQXx513yXF2X/10aYd7skzLBHolwy/93CrDjCPkeVVFB6NkxefBXMP0KQI1aFg/S9OY0vQ3WLbow54CZE2AtCP5lcp5SDAcs+2j2QWwlCjcYVR1OhDDSLpY7Vwy08ZL+Z3k7GBVJKj6eeHguvuE1BrGPOZ4P6EFuhA5ugQ12RWBEdpMYFEGDvBw2afTwj7mLjsKpHDwu4XQCp72iEWtooFm4n2TFOmgl+3E+E62P2V13TC9sdwisK/iYBdiI07wIzsVJe2USjLLZXa0tImb4FCVVKCer6wg8suh4EEyV+Ehepg8KCrrhTX5uC8vn3OaHSsvbEvaiAVyETP1cthM6YhZJrzI69Pwpa5jHlonLEH6Hhm1p9Wxiyn8mVLu43JQyU0XtX1+UkJyaaMQ0R2RhRT4MTG3E1U+Iz2FVavVEuAx3qzdivVg/weKDv4gBSYf2Zvfm7pHyCleJ+0r0LP1ZwhdrqztUuJux7lNAUU/z5vFSHbjeNjhIDuR994oLjYMRwQPQ2IJ3aXJnNXYcJuGQSEhF3/crj3toekCjQSvZrud8ZYHxqNQYtvld78ImTrfuijuN+Fr7xPzhTAAduztCnastjLmcsjiwEUNKkfeCIpvsanMqAJJbYsYUxxFgh37klk878i0l6CN33qG0RwLV8rWUMCMR+H5Q0Zww0zRPWGjzTKBELVadts9ks5K3dNmm+V50kIRyzeIF6DXmFShDQH2OA82hjnPVSQwJk14uA8acPhJgPfM1hoc1iniGv9MtkHyS6G3tJiz15/h4zsRhIy4+O/fKDcfvWW0YufkAOMxoboGans8sCS3mpOzuwgiBUkVzCwihsh8h44lpLo/f7GCBSZPlbu8ZZDn9nMXVttEJJKwmWQpzDv32FXlsOMmI42D8v5++7A8ahai5RhhCewHlh2dPAqR1iD35ODFEEsYKZvsDEyc5y2Jm4OBAcNS76F59DYuCIDB9S1JeS3u5KYXuTBlbx22xpV3MDphXqxJeDDEwkmsBbWExp1W8U54ZMhsbIPeGbPcJqOFBQfIC+BKHCs069WMApP3O4bnSpTzSlScGKDWgCCGy4pJt/eGLtUJCwy92NuYlCR5LOCTQ0g/SEzef1OAf+zd1Zz6cjdY8aWEBD8ZNAswIQUSQbsnwgBg6u0qU/Snv/MirM8nkxUDji7wsYrCjhCG80fz6sv0arKMW468/GwlvCzM8Avd4F4oqZjjjEvvh+bv1BM11nPT52pYGk/jVB1b/Q6VCUs9drlvNZT4GmIcmARPfEBRrXuJIcEQO+ljHLgfuM5eBMEEjgFYAJiuZKHw7tjZqvTVMzsvnnS+28He9R7MbsK5cL54jcE82HQFs3XKS5y3FU+58BjTHxvW4nhrZfKvGtB6nkuaUhOADZv3GGNFZdg0kICC2m0XtYP8oOWCDNnPy5Nu3GxJLKzx5MLZiPkBAQKsfRVFLF0t1Qt5oymplxhZu+8iaRgG25SEhBpbF/vBuzTs29JlRi/F243RSOxFiFgNXqX58uruRu2oBpAhssyGHEG4fG++JbKuot2XK5fGBq9c5o3U73dEBApJKFgpfWvefKAMzsCMGSBScsVtfG7a34j+XT+tJKULIg/9f/b/NATbPT8f8/0fXfULUpSz/4FdsLJBWwNE12byCRLKRAZ5rmTpIfm2JlOPjyD2dNWmma/iHRyW7A28DBrhxRutS+VFAskQIm3yojHBBjk8Cj8z4MmXK6lv7HyGpgVPwdUwqwss17LFkq8UuNuwRvTcyrpdksaRCCVLSbQm/8Cc+K8Bvx+WkPzVuqndkQbYV6j1tB1S+nbQwRXG7oMvaBrxOgcdxnMDLTZS50RbuLOHlc+49y2Lny+QOXXCFkjhHtG5sSeYfch3ABHTK6AHoOcJJfZtMq5CnPaLw6Tjb7hqvu9f64fzgdCGelRAlhgmtZ1lyTrM0k2hTFpm28jWhg3Ht0v9OxQYZLEM+KBpRPA96QndWENFZ/NsBvF4w4085wUlHm74rBPxEu97FSck508kkvq2p0uXY89sL2eae/ZvBhmbUDzZWYktS0A7N0WuCEzE1H1vLyqwd2kY44n9QGJoncj1owQtWJE0Gy5C6vD5rQHId5bCGkhn3DY8YKLRKmblOONp863ydD+yeWcle3WPb43S5/W66Z1ib3iIa6OpDde77DnhGlbbTx45ErFW0CEG8v5nhif5xtyYm/dxKlgjZ4Yt8KlX3euKIdr5U7x850wAZ0UqH6Hc2aCDJ+O4Bx6iaBYnfFNm/mby4553FCNogtAsC/Lx5jgqdG+bkaq4imF40G0bxol78pkrY74j7LDrrszbQNSvt7KCGJAnJlh2DHUWQKqCPiH+UfmBxJg/4ROawCCX1EirCF5aT/DN1Bv3NhJ7xhhOWbmGnnXC3A4FXy+tFwPqwQh9iZoupEDO21pgmtsvOXbiPP/WFQ4//GQJOeZxwf+0O/nJPKreh7JCV9H4/1UbU/ptgRBFByp2jJybIwEzPTgjTaAvRH81NGHMvkuo18ejaHtsqDt/hKEYPeyF3hRZ+LS+MhYnTd0NA0Wgp5/vPXhT6UcAijmHi8/lk7DL6HJe57q9Ur1xkJN7FydctGZIk6ZXHhWnRSzyPqsmYoibZJ8pojpoZL53VCnmFHGZrc7Ty+yNtQl+ur1CN79CIIC+wWJ5zV7q3qkrWCneG6hfq4HVZR1Hzh/4wtdV11+oalMfYVDA86gpdvVlCzM42mVMeYYoUeLYZ30VLkHo5J8qTeaGoiEslhEpuk9hY+6OFgSNXoXWrjV35uqRHUnS6aRAwgQcmbbuykZguLqy9axEc6Zsw6tAzk+U4spoLlzsRwpfYgqrq+ykkKgeZ6nz4To3bVfDXfYisXW2EbLp/jq2eq84TZcgsBMmj9kRD4ZeB05WjKwEsCvta7KEZgsWS3wTW+6YGwofdFhXLMvmnI3yH8fOxw4cCp+Afor+6D9zFyO+ABaAKTOY+oHTidrLLEjuh3qxxfz97YmeX72yfhP4Aci/SmpiZLbLS1D7DJhld5ocZT9IwCIv5Mc3WBl8VmiF1SvWbIzgI9cyS0JvNr4aL3PwEDzeI6MZJxBLJGbVsNuKLQVqS0juuvQMPto52UiKTXDC54VgCAVTNlqzYBFVMXxP4qIKrBNb3VTxfXrVSl5pTn3pLIOuXaLoB2nx9yRLHP6W9HVjoKFM7nVVZrLlD0RmR3MeVdXN9b/VEuWW/aE7bDJREPGRuIjYS8Zm+rzKWywTwjfWj7AVclIVEUXOPFxJkJzU0qV1uAqtPwoBxAYKMQlqeXR5NhVXmWOmW6ATBpxkevhgkq7iGU5XwlEg4vrgowtlVxwTNFa5FMNZsH/ZtWkJaiMwQryHe+33LQGneDU5sWAZfkEptY82FHbZ3flqDL9HGSRdEOOGA5URUuIvFpypPcu82JOhLA4TFn3vR7n8rXFxXwHN6hEtDoHasJ4rPu/X9Id7AiVkBVyEqDWm0WA1GnNufBXJhJAmlvHFBOrbhPfK6RmbB8Dh1KKIPHCJrpyAre1tpEdnmyryt5/wohZy5glnmbRSeHMoXdyKlA2H7PDOZ7SyVyv7maQo4v64S2G50zB4R9hIWLYEN5iamYyEZOg7Q9ZZJC4RE8Deq5flc4hPao13et36GXt5mgUadbFV3OC/mfCJ4zZiAgVVPDwmN9RYIohvah7H5+P9mfBdwPQNsol1Nhy1hFLMxN0tVfNvV8j7p7CuzsXk2X0TXW/9Ezv3GSPx7kbQJbenYYEvtkuNEmsyJYFgvAy+AWNxHcMhw9bRatpv6E7CW5BPbIjevHvrlLWs6u1ElxADFAdnhOz/J4k+FIMVUwakgmyv6GrB7s2Cj3u/+PvL2VS0XQV71YxXJDvPIlGnXViVOzxkY3EYnBaVtOxQlXhZ3HmYKourd7nzG85FAWYaK9vxyZeW+6i0aAzHDkJvF2qncCY97fyYGSatF9agGhR1m2MKcVh405hxAzBR0bVVf3FMcQYnczwJp8fvD+Z2E8kyqPln40byTUZ3iz1FiMh4xA2GvNhSwkpQlIZQAmoLvUwH8hMB8GlBtGM5b/B9fqxSXqLXwjGGV+nDC6q2KMl2AcJHCHBF7S1PMcJs9EeCoO0A+Ff184f2Wz8DtanjiwF/18qfvA6Ra6MXGPzspC+CoXpz3BNAELNb0q48I8dbfI0gZjBmQJ0XtI6jwCjN+e7hWy8g9bWOp7/xMrlfCPapjETKG7SCzR8Kk7wY2h/7HXTkBWOJlGXcwTDkWf2f4t4OG7VHb2kmMRUN5oPQxdSeD7T7f2/inVszGCWkd6z09UPKzJnLvF5s3R/p/JPoXCWorORlQxOJOrkqgNusMb8yEPi/5YJOXbzp1EomBGdBbOPkDl67/1yPFXxqiP0l3y7CQeCgJWrSTz3w2RUi1CvfGPTBcc0fIQ1F2WZ8ox5aWXmUMb0LR1ZPHigQ6GUJW8TO4RWj+nNKvk75hgD", + │ "page_age": null, + │ "title": "AI SDK 5 - Vercel", + │ "type": "web_search_result", + │ "url": "https://vercel.com/blog/ai-sdk-5" + │ }, + │ { + │ "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDELZl8TcoCcuJd5mpBoMP9vXxis08AkGn+khIjBqTvZQvXQKW9xVvflZxZ2Ieuhq5UJqEKMpTnqt5PRIecKBdF+Fhf3haI+V9xEFcrIqyg3AAFeVii7epQXmTKI/OIX53ha9Ovlh7gmiNMzr1PoIVTb5gDTH+54lA+CZOpr4jntYQKabYZSqqA4EpoRZKitkUJVbOuOdAqLlZ8UEguig8OQTSY5I1hdzAZ6c/4n7SGXNY2Ovn42TRlspOTnDFUY2UPpSz5fAfyradbr3fpJcR6BIXGNpwhQsf4v4rLEijOAxUwjzD5saxJPRizsF62pgCxJzHb5xeipw2dElxa6oFsBqFmN17NnQSJJC2xmpp+zWheho0nQAPZVrHAn1xSLHVjhCgPGmJeYsFTyAANA+5/WsqS1zzo6fWsmPbD1+6FSPt3EVYJ/6ATD+RedTUmMCjKspFaMxiDwop5RwzEb6R17X3lGxTU8V2mre4VwV7e0HjoHtKllbvtS+jqwF+1Mx7kufk/y6Tjv8LRJX32AjhEg0CuG69jnJ8aPPLFCB3GpLj1lHtucIZt2kQ9B8AMfbTimuRD4ba69xZdBPBR4ti6pV5wg3EhufIXiq1QV7w5ZmXK8uRkBJOR9v4eXLuQkn4P7OpwPDGEXCI/UrOXjufvvsMIX1abQ3zOpnR4hJ+xOy5C1WdFWbDOwHQhQcMTRNjPCHkQKYmDoW02adpNApxMOsDRR4Q5TF6WPLtxptPcrNgy9AWR17WrjZR7S+WZDdgwKSVM0ordVn4+fpd8hpDNW0UblGSVfH5FYA71BjG7JeDooIG06pSQPwTvIAk4XqCi0gMrn52r4ykPbDYX27+cgL6Nhe18tBCKk32I+9uxai8dlcYVdII9MSEa3BkdMo0uqwGiR7D+ebN9zaQ744nmw6vFCmw4FmkZBjp7sP0u7npMijuy7azeUjKx3+XHulJ/oIOk3eFgg+za6sjCGlvvJU0/mTNw5gctRd/WgCinpJP8rq6Egy13qHMgq5KHWejdZNuQR4n5/8tXvZ/EoEkmCra94avq54mkY8xF7/Rc1JrmKlEQXg8lznr0y0wX7vDdgVYgzT5SsbC1FvhhWEZ6uDVzi1w9IU/LY3UGkiGmxTkKfaaph4oNFRjPJKEvuXJsLhMA4FlM2fXs2g3TZcoySMrP5BYPIqUx3uZaXVdY6FaIoBsBttEpT9AnMJpFoB6mliLpRHawSpTN1xhvVjayWCgyKo1y1KDDOW9RpvvVwknX4nB6eqPBsZUZyGCixbAHvtAF0+Wl9A71QfZliH/Y6N3zNnQ+VXA+Bmowdu2ftr2H4hBX/lHTHMc+WHMIZReNyxP2uCryVAoI4p9Rfl673CTzywizbiuEF2clczrQ+wgMWlwImyI5E/lwUYfB9il7gpz8BoOhnM9zOb8qAfASuBvIfkU3G80UCMvV5vK7t0f7QDLtRmHlsqXMiR+zNcwRT3ciuDC2/8zWqW4f7MqsR/l8verPGj6TiLMR9ZqafXJ2ciU0s00HQpjSv/ikwDn4Lrgx+kCHo+ZkLHWPDZR5IVtRBI5lKoOX20D6x9iPtVihExH5zIfECCc1Qt7nOGr/HDp/qIe/K9Me20Ys7mZUrOsF6XrXpz+O8FNbq9LP8qR1lBHNzZBO8IOIXYl1wVEYB7ZWCno/M9lfMGPDYnh4VKbwJy7V8KgmpFpq7oUU9uW5+sE2SgmLpRnYXDwh+egJK6QN96O2ts4qPFK7nEuz3ymV00meDd/btZN57tQ78hioMPddXnvgdRrqcjPmQ344QXWLA3Xl7BGpmOClfx9mJ88RKndYtoMgDFsmeSpTAFCCKSXP1MGGiaFJC1Ru8BBTB3GjqZuN6AAHeTd4c4JgtDDE+QqMloF8qiUPCYe1T1nU3vLzfW7Y3Ua6pdGr6jJLtwqeL50YMgf5m10u4z5VAdRNb07bM+QHXvdf34tybOmsliCvJ2PzFfqb/nZCf3TSrf9Vj+Fm0Ddqj7b6ks5pu2QdooJumjaLWGdHBmxEFQ/ZF2cM8am/nbCPDCMNIndRioRV3Z5kJSIU7OniO3/KKmG03iZdqKDcgFY0E/yiinJ6YgiarLavoCGBwfufFcrc/IacAly1a2WxxPjbX+Qx1qSYavLtaOwIywTAQv5qdmt40z2DqeHwdTtBynY4zSpSse5Fvz8BDSN/Z4sUklFQsql7eTiUORS8jDrONMC5w8haMAOrzFiGfkYxd370ucJkgIQ9sHzhO2gGdrN17vjgqDY8Ct0LSxlDqICiK+cGgmiM4WVR/Zn23FXsKVUvHb+tbf16MExctgI/Ctm+v4n/p8KMxnnNUB5bkDwL0DPSEpqDoqXrcYi6kP7hnJ65UgSVnOf3MMJXDAIeY10QtLx3H/EFRomr+CXnv13T7kGVrcibVz/J56qr+uGAM=", + │ "page_age": "3 weeks ago", + │ "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + │ "type": "web_search_result", + │ "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + │ }, + │ { + │ "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDIPCrn2iOj697vEAzxoMTyuu3sH+AHxv/pXxIjBiz9d8C6vwLeSEWqtDDOZrRdUd/gEOL3FUUEXEUQr6dZnSueIVeSXmMYYULxALo4MqtRBiFWgv3nNsPkKlaHauMFgEgR2i5cKZkj5pXvh+XYgBCXOMaZKs2ZZZ2UpiJeCh+MEbvIrv+5CS7K09Si68atIVz/wsZNdJm+46s91+OR0Wtcrb3RWWaPoi8DQu2WneQwSSIXr1GcwT84nB9vsjQAhy4ccR35I/ZjjwWJ38unpAWmzyDa4DebDOPkKqd4FG/0HD+wFEig9zY6k38MDv7Aj0tpEHqdR3yzsEzhPXim6ZVL24PUef26/yuP3qhsflTaYfbBCXhePh0dDH6j+oPRGFpzDP+qNaeSBIxm7Z1pPvwa68PAD1WvR9QtAaTkBj22ZcHtEuBS5KruEaThrN9IIppwDXGLgMPYGROoeYUDyFFxa2n2t0NejmNXULIsqwF+oaf+UiU4DOJFpExZNpswdYCSEL0omhIq/P0O28PbYr3oW03oCR+Ya89oII6MDju8NwIpgnVWqgVActpSlJDV29Z+hZLbKznWjHFCS/QmQ14ph/iyAiz5nX6v6MVZdwsU0EemZzBR3Viat0SwF/A0bXepTspOiz07lN/gVDe6zg11eMVmm03IsJsF0+xcwP55ASPQz0c1NsIGw/3YUGltSbyGV2/QavthU1PL77EJMvqZIt3nDnFWV9yLNsl1bSN+Q2F3rruXLFFXIg40mjTbapynC2NJBTt7WJvid6gGJKYXSDO6M62/3T73/bO2tK7mU6XO6h345PG/uGIqO1BP+KmiGJbtAwXmCRMVrCwLo/gRxngoHRGftCFbbsZq4C7M8yM7+sZsEdB1LdjZVPHtwKrTiU+cYw62Yp8s5UBu2qUgqN5VlGNUhYkwFCDSaUvgBPTTY3va4qS7JabwPulOgXisJSbfOSWhxaCdH6/SHi45TCbta9RrF1TJI7J83k+jHItSU7XdlFJbgGcc59150TYrmhHkhpxJIJq9w4/BdpLywhTPrurNvlbrBl6O7XLfLEikpOgC7RJUefGhoCpBX+JB+33BROXTenMM4FeXVud4bP+kdXi5kslDd3XQwgn4Td3zt1D+Z5aWevIY5UUOaXTYDEnooxKH05rbb20w5XGqTNx1wO3+U9MCTI86vS8ZeTptu3S+EcHfecZLbLEHETv2BDaB/0TkVup+rheEEgP7qNxpmKarmvMXB9xxITHzmkAhjUi9IGKMm0Pht8Rtq5mq9J/2Z0iYNte7nFyZPTEc0vxiCE1VYAo85IsFcZHy/nObh0xvGXPWFVHj7LkWYfngxQ9KsQOeOaBCptP9o6d/XyLuxS6bzfx3KBTNlxIcVC3I9+i8+HIF2Sr20RpwQfBvefd3cjihV8HHyFj4C9oT8InlFsnhBzVCC3OTuwnCl6bfcRNuAz4R/8dwItMuBNGEBhFO5V/eHaofXfzOxriuddAfGSH+fHtpBv76N1AJn/n4lmKLNOa201hUBLAvyHEL4KMj+SJgu3XzkXezSg8swfWQtj/n01tIGFy6CDkiANmXHWwF+QN/IUrwt2NdtBVdV7ufKgiRWZOGvb2blBmLW1aCyTto7NPbLpKIpFk3Iph+TRWk8nHep3RIw+b3UT4Rd8lZgiPHTZiNf4ewWZrlfj7V40zQdHTX/oLmm2IGEMpymdTiUsHP/TBc0jSZl+BgSo4n4TBktOooqsx3StMCmPbtWEVR10bQCIbI//fgT16hvOJOjjmkWTUExmNf34PbH9Yc5ejs6/UAefelDupEHhOQQcJ5Hl7y/LqjAV+X1+7eFODTCvCEjbWY5pDiHwwr7LIyHE+O7CGZatdKISy5zUFFNy2wsDsjpjwlUPNwiUFWkWp0hMuVQabrYMpEhZp/2AeV8WoqdF+EdoIwohcALkUvLc3Tyfp7iTs70QKyi1j05r5sS18nkIQNmKcfg5cQO1jIFSYqJuzZ/k1N41tpZWGN+nZSPcMbdrv0GxerSExGRa5c1Z3n3w0QhvxdDWeEUQ/N2Glh+5Xw7dRWYLbeWOwqM+4xY97eCFcVKNHc1DQTAUf8oGQAEYuoxCxvUku/AaQoUbIeRjkZ2dEo2HF9EGlcJqihiNRCkOY1lB+Y+f2Vk62hzBbn4y+G2e0AQxeAALu/KaTAJMFW+tRSRYb957JgOncZ1Bt0eimOhVb+BcZJIjDLV7LsSsyr4TAnV3+pfZoAwrAFarPm9HzaWyf4hfB9LjC2shIjDWDlPSEXlMWijeuIHtZw+wp5y4DUq09dmMv0563aRxaBA5uCqYx9hFq5BaqZYbhcKavHB82GH2tLT3BsCMBY3yEdaf5Fdt644d6cCCPyr8Z9gcnTtnsuT77xyroW2rizHBuGv5k8cp8v8M2jbnBnoae94WP3BjO67Uc1UPnOOEZu3LZyAPAPJymRpPCo+I9/idLuxLrwH7Fv6JfzaJResFm5TDH5v5yg6Sear6LlQ8OJfIgFkTv5g4Q+rSFx8KqNKOCEHJ/GGEU8V14gpONoti/DPGM7huttkvGp5ygN/hlw6ICeqNF+q0tquUoBSNKtZY4liUH0a+zQduf9tUHcYbq6yYHPYPHQsL1Z0LiUP822FuG5f1uBavEjN15AhfMq1gzWs/Jcdc/zmqaELYfzP+nalDGVtWwub3fHOb+1/UKcV+utk94aAU2nfnzVIqwQWSvgL406PvPN5CjzxaJDCg0xpwkhHpgjMmAC8xymPxy44n6VaZLDpKNgQiUJBKN4/STwmEUlJftqEZsdL3CRURHCYucsoBs9OFfDRZNplJKQfPfWmXG3pgxr1ys4cNv+/kFMGiLj56qnt48FksyT1SjBgUQg6G3/thHO01AqKHDA7TEVdQOdCwYAPnGAM=", + │ "page_age": "1 week ago", + │ "title": "ai - npm", + │ "type": "web_search_result", + │ "url": "https://www.npmjs.com/package/ai" + │ }, + │ { + │ "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGYufEh4nD7p+ql5wRoMwXOXXp4nLBiAO2E6IjB8VY4Pd8kczLjSUcpep4BAG0CYBM1xLpcm0DTv5wNpdYWzTFw6hROK4rfceb0A3T0qixf3rJYRmxc0MAnOs/5zGbxOczA9Zb+iEFAp4LobVl/BWUywsjGF5tOlEfKgfz8nGEpwoqUd6wpJJ6HvkGGe6e98CzxK4x4XrdK7OV9mEjNwai9e31mcSBgvrqNYTRgYRKUsBr7Jbc+dD0AYX628zJu7VDdiojrcWBsSnfsemra2BU3KnmLzFKSd4KxWlee069INKT5RK2Z5GrzJRGYe78P6jt4YVG8bgyVNOGITGz39k3ktzPCph3lpT2v2pBqukefzuU952F1w2lTISV6p2K+tYXa1qhqge6yUITUwqevoL02OvoR+QDbB8A2dSrSWlUFyOyzpxAlOioUlPxNbfAmotc5LhyncSVQftFTX6UhNysipdBEIeYdeth/QWafT6BxHCyvzN82Tk1ov5Ixw4+S9QZycqy3fYjVXb4EfRALoxRNKb9O/IFT1ab3WrN2ROrjQ7rjdOeOHHO73ckRonDhrVb5OZjOjzTUoOSA9GaEry6v8/eEfo3makKr/jf1vy/3WnZKpp9zRFa6f1PU5/TIflX93TS7rSwyRDZs1IGWbSvds/CVvZKMTYirLZHFP3NoUkSCtFtooHA/7+kXNyeJfIJL3lOsyXEvKKCCcl3lhY0He5EdL4allQfRX9QGJW/cvKueiu+ooqgGoPpPgwTOKKVI+jDf0qv/oroOd3mxU1NDuLHGCU32+tH1LYGeP5oFlV1MwD08xNKd2KHdlzSwzDjK25H08fVsIPYgnKPxFB7X6F0+RLTIm05uRsQQX4iMZ/vnUvh21yY9Q/TyI5DuWKWS+2F3QUBc1ku3qgLJLR0c8wygGNmIOQ9eXBLx8pU8tzPguLesflxBMsdEcqx9cufYlVO7VRU8Hg9QpgMrbE78diwqv5fzHAc1WeTkdRbyI8kxiuGSbiR4Of7baYq+6EUE9IOobhwihTo588tIbPoJB3M8MoKWaiGL17ir7PHRlyemuxw2FsZx4/jl4Pfi7GsSJP4lZF2gAagXYVQrQ3oRk+HxQmVw24QQvRy045IfdcD4Yo4mBUl+o8GokSGs2QBlVpe4GGqW0e+vF1j7zRG56S0hT6oBKkjPLkzqMt0MceW/bM0qK+YpXs69WLWrGcU2ANome473/LaxtQl0GeA+neokQxWBy2sPcDvMWqhu4r7SC2+Morff85oylTXNaBdjLg2wz5RoMMXP3Ps/YJcxMB7IqSb6bXag8ToDiGZtMcx4pRNlzXhmN42ah0kEExUrHewCAICoVOQOZBIKlMg2ltTXfdaECXeQwFvqqhN0Kkh53ki4VaqunNAPlhdbOWn+/cYoIAU3muQZcOKN9REAp+kbLC40wEatbkU7rOxZOQ0UEVwKwgB0Qu1YUMLoUMwJcuOBIwHtszTyE6jiwU0VSXzcfOsmcbYBpTbR3oAUuqeEUJgvqepx06sSeLmZeUIP/z5yUvR2n4kXn/DMwOF1f0fAxmK1yd9N0s2BfRFH7cOSmyLBtw2EIoM+PuomzvdxkaADGs7ZOkp/fg7wnF2ZxkzThR1L2eREcYcV+QQyx3s9fRzR1SDAu5rPrjpF+p/iORBxhHwncXPRbTrFXdly6DgEKpnJsgiIbCaV/77CQcmXcPX79/+nGpPOwFY2yHdAzN0s1ECXw/6oUu02rHVv3mEOvLPGsA04GCVKvNO3R3e9LA+cVPuTaJTL2Lr9SWkt7F3JrIUF2bfO2Aizn/MpR9bt6gIuDTACfwOQZbi2Ht+xama3fzFcnC7a1FP12Cegp5Vl+kjBTCW6dC+EQ96IxOmSeNnGr9KAucXiclmw7TU+j7Xk7+dnV6MHcbtPvIYDND84ymJjYN1sgC24fZWh5z8e9s1tqy//hK+I5yQZ/K5OAbhwQCVFA3h25GnzFJe9XDMXkQ0kF8l254HxOY4HMhrAP/kNfmyKUM4KDa0xyRAgquazYc77BNLxOzLPCXJxdfrVc2yrGyA9Wu7uaJLMZlh6w9DcvHtsYwNXt0ii6VmVDeam2CxVec9GKMl3na4pdEdL+CFzle6usgCldABNvSKvlEKFdsHNE5ONEEQbQsOtFcN2nvKSpnF63BddCS2t0iJO4I/rFXtY+c3nHtZIBUIbvadfy6guSXo20woHHQL1cu5nKzUZjJhZq9pRwUV5FjY2qlQf4G6QDuWUODn0hdTBSOBT0mj7yHU1aKKnaYgj+f1wlU8b4xP4DDQIEugyJWKm2oBtb4OVRqpEOniTRlv6OMMkzL6fgLzns9OsEjBBOD2p7MgYoBFx0VTxz4v61Lipcu/njkkaTRPLufeXjxH8FfdrUWZtJwn18T5OsbUxwwJX2Pd8F+D41d5HtxtbNK1cEmb39lCf1JVv/WjDK7JknWMnbrc3wIutmJYyIeRXhWsV54UsZjIUCpkXWgUk41ZCO15HALE5CP520ZPL7lVpv/AnB5GX78c9WejKy9KyC6gMPrifnzDZLehE6dt35GvgHjcASTSKtxRowb4yxGF5bLtmZwSCtKGWY3Q5pwj1njmnry/bYT/7hMcFleV4yXvQ6jVJIiICLxoqeZMtPjDx7pfNJAd5X5yL/OGbSy3PKNDXtOwMNAxRe53/sNGRyy6CJ1LT0TjMBTcJG/RAk1g+biMOQ2vLzf3rUEVGjjRzEhAGkI+4w+a6q4+SelkY2nwLb6lgXBXlJ46KpvvxfdhXUzrRgb2fHgO8rBkfn4cudLV4v2HDoL66vV6meYgJO3d7S0AV6I779Af/itYol+9lc7EzmAU0wblnTMFgCEmHT+eIved7ASiUgjv8+2BUC/nK/Xsbui5Iyf2j3QSr+nWYQI67GJDWSJsZzUL096HtL+3lqpVccSJIs6i1rlyA3EBSSh18Pm5JUEsLbnJXW3THdUlDW8Xb7HKzlg7uPj0/ATTr+t8AeYL3awkYF7HixlkVepmgHZT9ZqLexQoqnIY7Mjcwqe4r9wcSBdVioLfBXVzBbNt83aeOQbdoPQi17/PgHWh/Rb9MAbLMtozHUdTp/azJzXC6p4p1zknBP4r//nbhPsJBDRwq8PXg2Re+XXOcBgghzmVkbWe6ko7Mzl16X7BSXHoE8r56for2YZi1qP3XVJS3ZAGhBfjbFI+5byq6/ASZnKnzHr8n3TDMJ04OA5TS2y4lscPaN3GBBuzHnpV54LHj6K/BRQpcByds+i+u+tzE9xOCbur0O/6QLPOyhDkGJCrXBjaAcUAKYMEvP+45qRZaeeptFWAQFUHSMKqcI9USXAC9GaU3EtK6FcUA1s/TvW1BjCFKdYoG3FrlLr4O/svbR65e9Uo/aaUKjYFW9XHCzID2JRgbDCSWfNbspud4djQceiebZQqJOWnYWbKOUbOGUAeV1ZSq1mCfU1Aiocrf/wJvnJ8/1/TwlbWvhowGyLDRZycipzZLCnUzHvIYQkwi+fbwHR32w5HnhIChbqcxMIeyS4gskypIvYOBtMe+nVEt4wIL/AXRXQfOrXxCoKg3kNMjfSGrnxGxgBlpr5iwVSvGypaEPG7M/KTleCugjnt0+gk+jYYFQKNJ/yo3A5bAYSc8Dk4cfd2n1mqcJlbzchpgKWkBZFdUeocy3n6gjbhrsutL6cnbXEw2R5kGeo/4zdkZ71g0slCFuwHiy+9XgiAzucyg98qAYyfsB4f5Q/5cOBqsXuDk1OOLJmOPagE16Zzb7O89utNHPN1tVeS7Lcm948oWP8B3ESEfMwCF2k1C8eYl3bu8lVLDZB9KonaPFDOkzmMsgTFAUK/0Z/NueGjBbeW/RZd9AKw7i0DFscG0H/k/Kp85DWfvh/IvL6Dco2cMGcq0VrYzkzOfAVPxWd8RP0fxTmJ/YGgyyq8SH0gI2LeT86GGsnP3Mg0koAIrDweFlpnILZkwi2ZorjQqteqFUyJZSavYcfluVLgejP91wEA3OF2spXYbNyH7rx0edYMVE/XCxKDBM8to7AVzfPWQYAw==", + │ "page_age": "1 week ago", + │ "title": "Google Cloud release notes | Google Cloud Documentation", + │ "type": "web_search_result", + │ "url": "https://docs.cloud.google.com/release-notes" + │ } + │ ], + │ "tool_use_id": "srvtoolu_019WjtfaZRafhhQGVub8SE5G", + │ "type": "web_search_tool_result" + │ }, + │ { + │ "citations": [ + │ { + │ "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + │ "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDIpN60R7LeMwMmRxgxoM+5FjmBcWwA9B2zAXIjDdpRBukpgg0ai58JaRSF44Hc5lJsyROMhQ+fdYMYoXd0pMEnyWh1AEyw2uC3DlLIMqE7gQZiqxxXOuuCkVV1PnqaiwHE8YBA==", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result_location", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ } + │ ], + │ "text": "Vercel AI SDK released a patch update on May 13, 2026.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 256, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tool_choice": { + │ "name": "web_search", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "max_uses": 1, + │ "name": "web_search", + │ "type": "web_search_20250305" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 75, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16216, + │ "server_tool_use_web_fetch_requests": 0, + │ "server_tool_use_web_search_requests": 1, + │ "time_to_first_token": 0, + │ "tokens": 16291 + │ } + ├── anthropic-stream-thinking-operation + │ metadata: { + │ "operation": "stream-thinking", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with the number only.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "signature": "", + │ "thinking": "The user is asking for 2+2, which equals 4. They want only the number as a reply.", + │ "type": "thinking" + │ }, + │ { + │ "text": "4", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 2048, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 1, + │ "thinking": { + │ "budget_tokens": 1024, + │ "type": "enabled" + │ } + │ } + │ metrics: { + │ "completion_tokens": 36, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 49, + │ "time_to_first_token": 0, + │ "tokens": 85 + │ } + ├── anthropic-beta-create-operation + │ metadata: { + │ "operation": "beta-create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BETA.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "BETA", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 + │ } + ├── anthropic-beta-stream-operation + │ metadata: { + │ "operation": "beta-stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + └── anthropic-beta-tool-runner-operation + metadata: { + "operation": "beta-tool-runner", + "testRunId": "" + } + └── anthropic.beta.messages.toolRunner [task] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + } + metrics: { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 607, + │ "time_to_first_token": 0, + │ "tokens": 663 + │ } + ├── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: "The weather in Paris, France is 18C and sunny." + │ metadata: { + │ "gen_ai.tool.name": "get_weather", + │ "provider": "anthropic" + │ } + │ └── get_weather.lookup + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01DgLUVPYJMvTXUdpVrJyUTE", + "type": "tool_result" + } + ], + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + } + metrics: { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json deleted file mode 100644 index 7b7fd46d3..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json +++ /dev/null @@ -1,605 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-thinking" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-thinking-operation", - "type": null - }, - { - "input": [ - { - "content": "What is 2+2? Reply with the number only.", - "role": "user" - } - ], - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 0, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 49, - "start": 0, - "time_to_first_token": 0, - "tokens": 0 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "thinking": "", - "type": "thinking" - }, - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 5, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "start": 0, - "time_to_first_token": 0, - "tokens": 18 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "BETA", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "beta-tool-runner" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-tool-runner-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 72, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1294, - "start": 0, - "time_to_first_token": 0, - "tokens": 1366 - }, - "name": "anthropic.beta.messages.toolRunner", - "output": { - "content": [ - { - "text": "The weather in Paris, France is 18C and sunny.", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "provider": "anthropic" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": "The weather in Paris, France is 18C and sunny.", - "type": "tool" - }, - { - "metadata": null, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "get_weather.lookup", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 607, - "start": 0, - "time_to_first_token": 0, - "tokens": 663 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - }, - { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "The weather in Paris, France is 18C and sunny.", - "tool_use_id": "", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 16, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 703 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json deleted file mode 100644 index 0acff38b6..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json +++ /dev/null @@ -1,471 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-thinking" - }, - "metric_keys": [], - "name": "anthropic-stream-thinking-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-tool-runner" - }, - "metric_keys": [], - "name": "anthropic-beta-tool-runner-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.toolRunner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "provider": "anthropic" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "get_weather.lookup", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json new file mode 100644 index 000000000..091d52d7b --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.json @@ -0,0 +1,896 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 27, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1416 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "anthropic-server-tool-use-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "query": "AI SDK release 2026" + }, + "name": "web_search", + "type": "server_tool_use" + }, + { + "caller": { + "type": "direct" + }, + "content": [ + { + "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDH9U/gq18g4Na1FlVBoMlJT5xR7eAcgKy59WIjC9nLo/ymxeFBFi+8LMQFEvGnyikQwA+yxiHHzAK8G3FvYKn9Vqi2KepwsKxDehPsIqzghClHq0zuxC+4QLFBYj8MpV2WNlY/7S5au8l1CCCLZXfQ64nQy0K03pbkNpUh1Cn+czm6XYkXnTvcxx60DHzyKfAgeh/gkerzf+Gndx4xH6UU8TseiVzoC0ynR2UJg0T4P3F4xgp+6tyYSwJHgT6OBHKeSKi1hNTaku0YjdzdHrfxG2sPk9V5vWkJTxS1qMiGi3GRlIsYLfS/K8zU1bMP19XQTzf6Qo4+ZzQq+4medpU1uE5Qjsn7SUK6QaIts8Ruum9b8a+GDpBYihxX/6WWPx+Vrzzq1d78NbUXAVjUT8eflGoWG5HEbl2tTIdz1M8Za6akzdMI0wlsO8Fuz4o/wgt/PUfOkXqxIngdExRmUuZUOCdw5ng9Wn3N1DhlCQQYNCRrmMDXu2ehOoAdoRRaoHbJc6OZlNAX16M/MpfVuh9cLX+hYTi0WJuU/E3Q0cdQW0xJcJWG5oyipVhdeuf7CWa59LRK/DAS6mfZvnqCF27QuWZsIxc5a8PrajGTXtD7JPG0lpgpVJs8EWx9L8h+lsPwNYgNsEErQIIMxA5GDrchHd/PBstxFU9nDZZOlMUe66yIu6V2XCxXWYETttEpQ2UOgtYMelP69PcvHUUfNaHKVeOcv69Zg/La5xPV/uVdniiHNK55sKDiKkYiPd3dnZ4pjSdRq241WHeGPLaEczYLP2hdVeNzCU2jUGKQtYkrAhHfmxjL+k8pO0yNP0U0vJ4svn8XL8f4OlGTXxSTeCPrRiPBs1mC9LaNCqSDBkXhs882EVrkBheOC/B/BUFocdiQptBagJ2YQ6zkjXkElIHxG7dRNG0cr0oT/P9ip1nm+vKN+2Ql5mrDcUETruFsUMu4WbVWWI7Fn/jpqa76K5MtNYe8k0B8OGYqZADi3ei8TjGdzp4AS9/i6gpsJ63GTPYLSu/teZQHkOkzct4/13NF0CwMaqmlsV94YcmKl4hfd/2jGNB8jgWImeFzAcebKJ73B4dX2bGdUiFab5fiQ4Y2WJ2aG9WQs2YMzVqJobL2kZj8dbDTRKzMm4AUuoAQ+RypG6EHGaIlY3PQK7X3Z8OOYOzblqS0rl8hpe8hoa4US2TBPkpzj85nROCZ79KZ4lef04xW+k/NNRqiyOVEszRExRePhAYcTnZj28N/5MxPp/uGmhZW+VzDlIp9cu40GoF/TdhwYMShJKkB6Uuj1wZOEKxmAKPETcRTAXDDWiCMJOxuXJjBzHKWB+v/OUepwjy8PQ9CjkXI3ITpxg8s3lwhGc3ZBWOwWpUVTd+/Mv8YNShnIIXwSrcVwIABPlvklGYvNgkQsf/IuJBoLk/oI6Sy4BKdvQsBODgJjQfnGyuiT+jAoTddR7rDKpDJk+utO4CNdLV19t47sBO7YB2p2sjuZp37onJSRnWwH7zDVaAuA669xEb1u0GXMn7JrFwkpF5gf2Dwlrs1Qt/l3tLTmpRD6RJ0OelL1wTVKU6lYhGAM=", + "page_age": null, + "title": "Releases · vercel/ai", + "type": "web_search_result", + "url": "https://github.com/vercel/ai/releases" + }, + { + "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDF3DY5jjLb4AHICGghoMxlCPx/T4A6h1kyaQIjCo+niLHnZ25L3/f4ccF50NkLy0w3wJuEGBeHmuVBt9gQzC4POCkZ59Y/TY4c+3/vcqkRk9xvu8AHT01gVWJqNiUyzlYE4w5iTPach9rwjjP+n43wfJBiyfky+jXzepIV2ADLYYt4F7rxBfYkHAaQuXy0dXUOZlq4g5Dr/ojVJafscugAabDKf1gTg9jYrZQeE8x67MdJASrrAov3QPraN9sejiMmTrENHTQzkkOzOu4/uMz7b05Vpm3YlVn2735GunbupA42t/CLtzdeNZxOq51CuUBct2ELj+sZmJiQatS91DFPGmXnBQOSBfIYMClhVTcYbZiSoBaa8X0MCsvmz3nX4kzCRBGAvGdH/3ckhjz/1g4R6/2DOPPHxOI8gaI13wCnVCAL9WgbqY+0wq+vd2orisYemfNCrlZZ/PI1YyPn3+vvKwMkI0CsvoWxyOtehurxcGu6mng4Pqk+Y2QKoryfbKvcsFBIWktxEzEzHrdh68Pk06QbDPE3Vtx8wEBxsLfmSJx2k544E5yi4IabVN5nR0byucbxQV94/kJu5XrvCmWFOR77d0PK8KrIYEYF4r516sVY1xd1n9EHph3yirtuMpbBMU5K/xXqSHDWLhGZHNe6DQJ3B8EbUmRBgD4XslpVwXaz+oAZbmA4TvbMV/ShtMCEpOi96cTp5umEaHPWRtucs3aFRWTKZAYUHdvM3p6cCvw6M8DpvWawWaOiPawLxf1nGgjadznVoAFPJmXDchheDMf62hPh634/YP02vRcKOylhloVvT4lifCuepZhXZRqsyFhYZCKEB73XlrV0KiK8TRaoiZsdv/texWfkO7bbXP+vcSRCQBl8U0PKUk9vDJt/me3BZp72V2o5TdizXP23FhF54HLFgWe5nH9ktzNzaG6me/544kxDWsXOJOKoGL68FiEnqrYY9HfgKAflv7FpCZbDqIHZAzBncSraUYlja7w7lIRgC9fc0gkG0DsMujJtGfhNvav6Qvpryi9hqax5W2hMj5HJTa9vC4G5LZwWisC04NG2590cFgh8CAlVxF9Rz6jLqm9SV9RkuwVCkFyUwQjWjlBsBOHlkHnSHYeIsP+gBot9iIMUxbob2iyv9txStjBoJCzbkXxrTSBK8WiRcmHEVrklwcVB2fc/YzTtCm/uTijjRIWncggm6+STokJ8zDb+E+21UD0ntKoMhbUOp9WJGV9kSrNcBHB/lxNcuxFZijRc3oJEKF5lObwPGqMife9DoFCoyBs2Yuhmwj2qsTNExXXdBisWXiU6Jg77+uEOZlKF3BcBz1gTTpWRRvYISk5/em8zV0QPIYLXZ3JuIbSOenWWKOOeQvwi8wipV8EJ4JnpCM/1tKBhcmJ8oW+d8UiEV9GH9i/1maiEZkbiy4CEtlSvQHhRScE3N+fD5ADGCg35hvIQQ8rrKMD6okihI+UOtd8Z7r6bS9sIm/XqX8Cb1Y3CDYlSjYkxO6hZAMny6F8HwUz4dhIJdf018FCs9rfA9AiS+HcZaAn+PL/mjN4l9tVYR8/xPolMHbMJtBAz10s5ppuBUTM4atpgOdNCjbYgGBAOsr3HePL9265DMbtMLviHcM21elo4qMF4m0aZ4oyMlgdedlgs4h30nX1WtAWz17+NzFrVjaDHI4AuHRFA4O6zvQPP0R++OzQ/FjQnUh1TbdkG3+iZPaBVNWaXk6ftOWIE/tczU8CSLu0Duxd2JaHMJIUnS8EVHt2Lp7wBUG+yLDJZXc+kRqdJdqchNzettzxGWNwpDKCiL0bD475rRs9dTruyYh4FvGl0/czPDsvlGDe/eTQRqj+syAt8LzeCJzn5pEwwuetn4g5rkTNkgv4zRBotc6d3mOc783zeVPlDcIiE1wvyrKtoYGJ5sGpzEIGrWn/sNOlG3ZBfCsvg4WqqTMmNmcOZQAUCj/dAC1QjFmz6M+LcoqtvGzASQh2nbbp2lJd5yrvtYubUDICuizNWKwSemXRLwHAghIwCnMm2wIvK6dvcnwBTpNGba9v1U+CHIl49zdq9Zgb4c4Fso4a1YKqGXiwqLy8rH9IscdQk1cA/WYgeCcXNJdLYLeq1aWWc78Ljv2gtRmCqKCq+Ykk+TnBr69Z8qcvMUAjrLHKY/xYaUOZr39KaPQDYBhZEn0yLl9r+7XTPbahrotPOYfU08oV1XSgEKnMsMV6kkPzLkPhRPhJ55h4qoVQZs+Ri+05lRvPkm2HmMSlLD2/5e7rPMYDj1XdPJWQ6+M36F/paezQDfMNxmf0VbAzKGDcmJqnuPLmuhcCY5MiY5+9ZHsJXq+9WTKnRBChnTvr9nuYkFMP76p9zsEWH7mcVY3tjuo3mBmjSxEBX1lZpesCTvUREKkLjsq5ICK+Yf4cJ1TUlrsUaqiLPDa8oZY1LFA1N/R/ylqd3xCbh9NO7JZwfQCPpRySvXXqe5t0yrfkQjh5ylF58FwP6O1UACb4C8M4JKQhRMR0udeB15I8tCmTa9S07TiuZas5RUt6yF93ywQdsFjIG70KGvrk+NUA8LmbtD5n2XMXiUfqWFKDUhVQPYd1XOqFOZmw1vs7K8T8uaxTeGuWSuhtsvA0en3Uey86JQ3FXI4JfpbRGK5wxXwbHX6lxgkxM7TS8r9T5Uxcn54HNaIB5hA1DtmkQAP8K43PPdwinMILx32dSe1KuQNDguEHeKc6Hj9S7LfJwgNG3QKjmxLgQblBTs/SKMZHGr2fU8JHppeaNSgWSZ0hfKTdrIzmeJ/+ualHTA+eJXdt+woqhIP9K1rvvCy399aRaTPXoHFvGw3f4p/CunDsFt7xeuIbeP7atvf5/UnLeXRG0hVD3GdeksfFpI0fCRkSfW9EhHRQAkTGGRBzilXRqmTB0dtMmqCGGyDXPRu9DkUgxeau1/ZDDbjEutrKG0GJyX9YHLbmDFKwSTnoeiphD4getBs7aOjs9cfjg8GDxQTWQ6AULGJva5z4O9yfM9jV+drze4PSPZJeBEtfyN5CUPGEWFD1Pvfxs2uVEZ5zkDR6KAofXIzW13SemKDJG3f/wYRtrl7ynOd9sSX1XwdXn7M3yMf/+QgWLar2MXJ7D5ga+9QlTrrxF+a0mWED8ZfDJ1P0MLk0pHplTFYfZth/XARrbUbVgCmX7lmkXeN8RCvm34jUuVgGAeKeqWhj4/1j9xPP1Hr0Zq1RZXtVoPusgFS7UmJwnOzs0wwI4a5Psep6nFsLn+dJFpgSAgJ4P0bYsOxXnSpbZkXbNlmVYl0Ntf5GVkDMbf0vIbLQZtOIe3ClraK11AGh9M7UlN1DDQ0E4o7I3xKrb5QCjYAD428XdKOm25sIBuZUZlcUqKF7diU3qiRs4OPXTWG2MYC8G+GDL7BjE1Tzr3MxZ2WFQ7jHa9X0MvZL0F+7spnX2aht0oJ+wyXakHyxgqxqn217FYpFrcaNCVew9EtU2w4nZPBx7dUFX3nhSWc5knr9D7oh1EQmwfWID3pykAQz3jFG1UXUeaUPkvYFTMCAIqMSlNMuxEgXC0kCM7aSzKNW44GOWnWl7GhZlppO7CTJLx3ac2mhhwP5q5a0qu62rATz3V9SCpXQPKmYpq0D0a7hLLz6tsrnm0I/gkCK5xTlwKsQF9n+w0Ha5nRcNFBJUEtgropRV+A4N3dskVeTgbzRUoO4LAkx1Ma04o5uek+sFAX1rYLQOYXFkY3W9k/ZeDsr4eGolrjC131jlRo8WcVt6stR3dgIY3de3armMNFKkm6ZQJgh7yBPhJivs7vwpqw5SCDt7tF0DfxJPY3rv0XZU042kpbQg5U7kjngI0oK2uB/NZath94q16VGIK2yME7bBylsPhqg4MlHYzYzTjoWJpLBjAAQoFdAOl+QbHf7WXgJ4FAyaoerw1uGfMQkmA3DwjEFS4x9qsSO4+JfHXItw/3mvt/gHgwhcS9HP8LXCZZeOzStiLvoD7otnHCBbECAa0ekKfqWcHRgZgiJh8SHHrZTMHihkqdcJbSE9n+XTpgOa8Cd8yAaC4FsjEFxQp/wfzBdC4GwPrpCZVriaWp/lYAEbihZuM2HYE7zD2G9p1Sfdx3X51B4JiplWDHA143wtFoscbWuhhkgK8DFAsrWR4Q2MozJFevnn7Tqav3DB1FkgV5gkgwFVt/ob+6hICOzMVNvisCwTuoFtMbgz2S2H8n+7xdLiozC4xZuBPSfndqlzJfHRRQpxEKQc+ukzPrr+v206bKchgcT9xCNYY71rn4QTpXD/G8scWdk2YdGc33wSBIIFKe7zeGY1LBbDe7V4aMskKZqOMdxsSflqJzaIsFJFWJj6rd3Ij6LVLl+URaKWAUjYhSrdxCLXIfJRMC79xDxH9A/2lzyCZE5RpPS555tjVWGAM=", + "page_age": "2 days ago", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + }, + { + "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDMBv9686WpN4/798eRoMkdYeQ6usG9C4FJBOIjCIc+6zBd0v4jLwOCFLej+uCHSZCavqu6rxeKA5RGFoioiBDg2IGtTfDB9tAvI3MlIq0x4b5+CGRb/YgxUpDj2lSSDlSJ9PLYMDy9urLLfoqFSuCFO55wWoIo8QuDQcKfIZ8OFB6utRapLUYkq/gGOM/c7Gr9kup2Ot7scC9LE2RkfyIEUyroq0u5mVXp4SRZhaaUFvW5fxqzH+C+0uaTuswePRO0urSNCO1yYdK/3y/+yDItaqyPsYTmSz4T5aeLHBJ2ZRMdHX2m2msuc1AAWxDX91sGAeTfGihfUVmgnx1P4zU4yB3C7710IZBv4ZbhUTgZWNq4MbQWv7ZeVSK1y6iYi7p/3XYplttfpp+C/nZhgkUij6B1VI4WtsIZCpJKIlI35eAp3mwxzv9fHRoAipijcv0481dmWm093Tri1W+ILWv+f4FqI2RDfdOo3ci4cyflnEqZJFewT3364sbu04k2S8MS7VXDEyG0+o77A5ZeNdXEEle2baRea324PnkqXdDfcWFjW0b+CNwjSvDKkWYFUuC/5dEcnIirjE4WO67Gm9vbc8QjRGiLOslKDDxyolHItrwXKjm8VSZQt+nN22EEjsWNKlF7u66aGYONqjUEVPK3a8pagFV8sKy/A4y7mDDgmoUB3D0JMrmDPgYY8AaEmzWNkEMfIAYpihXyCOCcPw5EqiQc9Ja7tTIEw74LpeAYZjZJQi+/t21KrYQmYEqriaUUY5+bJ5J1/VKuP+tqT8/1SIE/UeAfx++baR7VzruMzPAXHa9cX5h2vm6HbWzz6Tk2oAx24vXSP82OI3eh5owKZe8ZAtqeRQLxElHj9bNq1iek3GcLy6a43Yko5uZApyENICDqQaSw/eVQP4XcQQR7Bf94bj4r3Z/RvQvyGEDwH+Buak0OetPH2ZDUjp6yRo7xbguW2MohP/xI0JyMNDMfMbepG2QD64/iVX7GjC3PCGAzvJxVCiyEb/OqxMHnm5HDQLWGLYDUDYlrXdgZnatcd1//RLSGT4sCRraK+3p7ZmAHNkylBDVxyccg+TFvs5REpB0X6P0h2wfIP8azf7Ctq2FLdSBeftuneKbNHvhaqKmz0Mx4GXWY8inHtwu4fyc2y05spDVWfIToNc9rAxOQZFJn2NaprfQf9QTG5qlkPnl4VYHf/v/rDjyJYxJeLM0a0qLkqDQRQtOdOs+yUiVsjpJvR+uZRhA5DDArk+fhWMjOFLuDIqztXozf25x0wCGzDey5+NYdkqL12X5cg2/AHUXCZGYTOcfWJrG51sz6K+TuKqiXdTpU6rKdjowWNwAVK0rLxfEOwIHH7qoqdnv9i25I4rEjA0cTJavNgjvifuhFGpZlakQ50UliwjUithKM1BErP7h4UMkhYCso2/XokGPoMrV271pe229V9epm0lF9NUsphjVWUFp2tfWTFEOXmsp+qHSDn0B/4/aES4HFYYmn3j7/eDe0m2dfsLnxf2oubxPUum4OWUqpDOPx80YMQUeE2nPDZC0ZNsi4PgwD5wUcRxY56Csuc214P3zKRfXJPb/Jf8eNdKNpshPl7Z0IjSpyHKApVG+eb8gKQre4cRzBDPFRmYQLk/y/570CnBiYI+Q9i8XQYG5dTwBkYKX9NaGC2nY9XdnxX8z79NyAw2R89i7pikixAOBfK4BQB2rLwJqFaYLFynyb4wp9tss60B0nTJUQfIby+zJH7bh79tdnjmR1BM8frcrr+28rA7QV6Q+Jv4exQabpCMrWmdZSYFwqvil79YzUG9A+maHRtznw9BKDXVv3+pejWp8VwR9991NM/wPDinSKzcoRnfzV0E82JSYtE/F+qcNvnef/LX8XkuokEd00ljV46RK4gwZRIeGDGrTMXg27fs7yvPPdbm7ijW+YpxrqIAv1Z3HNXjOxCPmPTpzLf/+YFMZ2S99CX9bNpRtoKknyululW+McERJxJWKwKUVY/kJOkelftch/Df57/b9sjXqEVTm17kLCm1zdmQj8h+P5CFA3lind6JJ8jIiSa6WI3/p2wq4CnoFKiE5KrhwWuso7BJ4qMF26oGZ4CwRTaVEGgWxe7QCiWRvfCPJPinS5FQfWydYh4RkxqwlqgLh/EFuRdNKbQa03PtuMXXrhkihVbYHGus1oc9QBGe/FqjArpQNIudE7SA80gOcLxSd2qd8Ecjwk5+3IbyK/JcnC3AEa5MhSTboAjpgj4PG1rnZnU4Ua4kETCr7QLy2KgnIS8LhE1Fy+D4OwUdA03gp/xIwseLVZ/p2aHeiSt0mGRTkqhPXUVE+eGp5uDEsr4eXo/2h7gljilQkCz2rt3JD6SiaZe3PKYk+dMHzAZNT4AyDg4tMR3Ec31r0ZjIRZSPdk4brX8pyENNQfHUg+cyW9fV8qVMZneW/Tj+w2kS5telkBDOloQs9nxbmtnvyZPncOtdPf06xdplhDSa2NbbtTCmzkituNKInZVw/wCtGGLCoiewgHl8gMhZ3at9w1V9T2kQ9KUIMMJBf+qCI8VLLoFqWQcssCb/kovAepQT/jWCDAqELm1Y7dcdRikdt5ZlcOJJW5/ZY6+cFDoww8neMZqNAj+9a20vKii/a6+SDZDnDPNYplZn9bhPG0zIUIp5Jb4n3QhGV2kAxr3wDHhG1YiMyO3mpgh7zQ8aPYWXKsqUNj68lZoW+3q687Fma1TDH/BAth0mGPihc9LSJkOfcMO8yYqHJFv+hFgriGjFg3bqM1FcSgnUVYAKe+39+jVF1KorSJ0XGUzDg3RIGAOq/FTvvKNCfQ8ssTQVv1uVh5pVsdwdNBaHz9cUnHqoEVXsiXMaZjkmbT7vIV1ogNDe0pQO4t9ikvRfkse5ICDm9FW39a/9qJuNfduYbohDS6deC45R2UdQrA/HViVZFp9Dox76TzGvarRll5w0MhZ/MNbxUhu80vLb+pkQYT8GGvEjf7lSzRfw5x9WQmktWIN4aOeoy10QHJaaabEcV8M1Ymq2IPBFC8zMYV1upD4rRsNBx4jL/1sd71e3TW60ga8r8L2+0bqyJpXsbvtyrVF+jDUzEvcxvEAZUDMRLRPZcXf1xFHDRr3zRZ123P1+ceeMgLShKRphgO7mrV3ij5INrbJYjaURpl4jM4/EAvEBrXUKXCv7TSLOG/Kxpsqr824tU4tvjbiZDxfsqLb/n6Ixraow96m4letDFWtqdrkuGh2Nspbo7gVjiiPQolvMspQcbpSnFw9wiiMe9OQc9l/hyvA1otihvIphHoyLkiiGMJd0jYqrQWwo1TV2mW9vrq+S5k82DBKtRWtVoc0MMAA6XKtwnFUtIlI7rHpdj9xflO066dHTKHzq9Z05Y9hfCHPWIPgqwHqELwa+bvsoWKEbZpEAxZsQzCgumx2P+hiU3Sn2omdP16Bzul9sDsVn6OlSWg+fAkGPpPFNlkkoVa1ozPKkybzqLDCqmU3gVd4KPhjQbqh8RcoHpOK8miH31Y3Hy2y/Q/RyvEXAhRFePXn7VR/QSNlE+1YEcuhsAnnkZtXgtMCrJSY2/7/OUzzF5+GI7wuQrJjF/WTs0aYTNp2pnN31oKmwDNWneI1owLMMdxDTdyq322c6IkMElsenXk+tHd8a5k8b7Vpld+9d0FoIH/nIF8Q51dJmw6qRYq4WTukJPGnEBNbTrs0Tnc0ZjdzKbmQUT0cfpP9+xexUmfGX6pVuBT6dMbFKQVblCtIbaegBK0KaDWtlElHJ7ktwSINRr/kM0mTd/CSzxcwR6nzdLBfnoegwn1vOOb4clNC4YRuKF6+gBAGvsAi+bLNOu7x3s1DqcIHoajn5SHXQukCY63+RSkiY2rMMZJrpn8+6sbV0iX8NXabQk2Yn6iSG8jegCd/phV31IobDBXD7BfkKIq+t3kJjzJ5QTeK0HYeVFUwB6FvJcm61ig1dBEOrog/Q3mQ35hnVlnAg1FsxUMsoawbsC2goo+zn/kAdT1AaxihYRjZ68VmBLTrMFgCNxBb3CYgBO49nhUa+hsgy1++oYGo0gvFrNiLgWJjG1I9FImHnMVhRCx+qyAkNu3j2tt4FkKo7ETBJHeblkXS5SlaU6QqCOt/d3Ma4JVeOyZEeMr0zmNlQjHMXSo/2nFlwb4D+IgYu+KCEx/86jsS7pSA8lC/jDd02NiBI5hGWOcZMNi8KqSzd9iSCTUfpTs3VXZsppOEZOJeBsbEUMSwPw4xpB4EtGHeHL+oSO/pFdPDLokm+tzjxrTnlrkBCRfXgNWzM33AwzPTegGIeQnIga2zSa8qCuibaorUvjueeF0afX802A8aHx64LLTaZxrpHrX85giseF9nxUHeiDm0b3FphNzSqDseRUGUvJjQXN8FngFI0yhfayfEZDuxKIUThiNzaP3swHfHMTi/zTX6RaJ/kXm1mdh48bTY6IZo8PwsFwgReEmviX2EETQgBJSarzM+G/J479JZrukbSgwk+Pb6qUUA4zPGfChLC7faQxkojIr0shjZUXqHpspDwSn2rtEXTAXGOcm/pN1CJNZ1DpXoeoxNZt6w1dghZGLQ0FwUeqPp6kcxRkPewT4y23dWpOxiM5HcKYbiotrr03kvQo8IWkufmQ3tpgtN24qdVft3nqB1GG/s4EWkPwXFM38KspgHvK2bOL6RtZvMj0G84rQhXZ+b618uRuxUYRvPixdICleasobCGStJHgG7sPa1SMnb6eW5zNXnMZtkL7ibEpq4Qy+g+p9AOxkNMt1o2dsa4wHltm+omytzYskZUpaydaI3KZ9b4vVJp+ND9g95SjN7Fr3ui6l9M6spHEespvSXfBGSfR+JAS6SYp9lZ6mLratrRI3iTJ+LlJ2QM+JAbjngub0QVm77SyIiLicDNIZ/OZL1u0vAAEUTaWuvfwk1LraxmiJDwOjELqw0al0N/hL5lcP+6+K/qIEvSveagn3h5MdlpI6oMlWdP0KEQqVwHLqWvwrTL+INAzw3rABEmIkmqMXtzuT/b6d9PBtqJhrgFZoamO1XNp0bcKKqBUKCDmrfsOdtHiGauE1axbf9pisFhZYH3N++NRb4FGV3yXSPLKANU49g4nc/eBXxSICPYfi+dveigI902xRr0pxF5TJ+fyVWuG1yntCc9bsf0cGJCzRE9Jg9TVoFyWm3jpjUHAjWSoxdjeH4rcasq+G5AXxfUBUa0KfuSV+tEwxeLPhp2SFD27g7l9zBBu6hK31VTM1BdFRgJCuVWzB+MpGMMMdGqKxBDyMWnd1wUHqUBPnle0Grs1bUlLPTR+gEGGfY1iEOAe4L8Y1l8bnCzjf1M4Q7unurdQWm1/NvHwMCLIxgD", + "page_age": null, + "title": "Release Notes | SAP Cloud SDK for AI", + "type": "web_search_result", + "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + }, + { + "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPLrkGNo81n+yBpyyBoMFUod2jgVEruDGYN3IjDWVDwJtkvLNYBHmagFxyBmjB5ndX3y+MCNQlxBRijGW7ECTYr321pqMGmtqV/LEbUqyhSljjzzEkkPQGdOBq5lBMGUNueTufy+Li2bk7McQSm3C2lpazpWb8bAXShoTYRF4bJ4g3uhnFmv/N+fuyEhGVTvkD+xeyX9p9x1GZ/7+j8asP76yJ7/ns3Raq9eQacoXEdkzU3IETTBhH420hwajIEb5+FB3lOZ3IiAyJtBqjQPk2HcYV6YNQtx5QWvWLiyLNcADfJfxdeECjTvZc0+Lus2FpC+U7e+0AcMyg5KSIt/L1aS19+1mOHa1gBq0rWUp8PFu5XBo95hqE/z3xHYC9BTwscOEVHfvnfaPpBUCtmaR0QhzSFBDbUrPMIZXvQuEx07rNxIWccdpuuj00XJG8R8wRi3+0LvKYO65Xc3zFQWV7iYp6yLGe79zpoR19bhy/1CCNH/CAERmJBQDpagmenuTN8iaElvbN7UCtvIyO9yMuNS7OtMAmgCYB1cgmWd1qyL+ky1/cxEwllAyUi9rhK7kBNgf4KR6M2l53vDpjumoXGi4MXNM7CxqMjeu0jflRs2qZzzriMdHYtJRrKUwKdr9AUWFRii8hnizWLity3sm3UaVShtrkDM+jnZLoLBEle/Ql+sm2r+Y5JJM4ZcQzksvofIcX8ONEjMh0jVMotCO5/2UW4+VOO44DNc2yHZZUilKCmY0x01rs+Oi2IxPQfvgG2y3uX/SoDbb+tD9JxIVG2j8JmET+gfonD0IipNeDJ/venrZTDYZUdnY381nPnA0Bfzyj1LjeyhXdrlwb9U/Nfn5WZkpo2nK0UDYRM1xb8bQwRE82CuvFpDcS5Jvby/9X49ZmG7K+ic3dy+XRYD/vcL5HnMFxXGRuUOgchtVVNi406v2InEbhC7X2/MvhnhBfsCrNNonwKcN7bz6lGZhHLDNpG7Ngh+ASjfDBkufvDQ71UaD0oENs/TTv5JH1u+H8nFi4AnlnbCP/NqQlTmOr7BOyCnL2FgUcHWo88DujaOtAVUyFmyU3xb6yZeX/xpeRD0P2xOINWTMV39OD/fCvTaiRcBW4e+0UjwoCJyXbjf62eAC/dkO33bby4WQBTzymk/HCjgmlbtN7OaAl7+BYLkLEomz5byG+P1mHaz9T2qzjPEsBHD7HEC/jK6aI4rQ42ir3zAXOs7+s71dzZVIV+Pnvqo2yl5m1EckLRJmDuKdVcYrucvsKMqkeTVIvlzZK3ctNeI0S5lNvXTYLw0RUl5c3z0dDieOaGgWLO9DBN3xDWzLixA2R9jDUAHO9NqeVaahT6m3FLFpeZK8F397u367hcEXfSLmZSuIe7vdIcBzaLIDkbTD2qQRZ/+39bDIxM6yYgqbhz3gt9X5eyE60eXH6j6zrK4YnPfIbz0JUZ+3r80eqdVsjkSbdqDdUPt9INoHMpmzx4Rz8itTQGfONi1IdRgjb4kIZS3HNAjGqlNeGSVMO0tsWhqUpDWkgrxwg78fyctAGEQZr5/W9JJgkxUg9zQXkIppya3Floso2YNFGhS4fHFsnAYL93bfsgZ5ql9QAqpRQHCSK40BKAqAku7/L8PJhbrK8UcYr0SGvfv4e872nSqyY+ohDUcKT5M2TXcPAjPveTZqJHqO5AWI6ZcTxa62lZxM5YNRoVhpvQfELsO8eL65SoPi9MZiyUaj5ay/byl0l98EXjWxXxVgEfiWfDtlw3gp3gfE2zulFNmsIG6asfg+Jgp6a8TwZ2Q3MCgomroV/EYJueveWZQl1YpF+HUo5JpVQ0ZX/bS3oyEBOCgZC55kY0mh28YwGnqkDXqAXSHGwdC4NlYOCUnvFysAWlmOrCeUrzsgnWAOmlRK1r5RmOoWsr5nrx5igwWYsyddlFdsbxAhR+wWC2yl28kC300RN1R6gd260kr6fuKAa4mqoxB/eLKOvgom9D+YsTiORAWyZAN3Mupf1thV5t9mX0XxfyKndBMUjfzkklz64oidLF1/DBReFv8UVOjwbe7x+beZdxExhzKRmKqudr2lbEYjmVxYm7kwTKmKy6VVsF0+g5soa7lc3mGkllFRbvCY6RmBblcj/z1mZtK0Y2vwMfPtW/EkuX1tTLSuoXmF23jmZEnUVbOOkoqe7sPU4eWwRICuyjsUmQj6elbU35zQ69zMG3IFBVUZtOxjp+TL35QbD9M/p2/2vDOEfMOM5hBg7q0Di8rmxv+wUs/GFy1cILnWSvObau2Qp2TQcCzhzT6lo7fXxbRiqKC95RUZOfvD2fUdL496LTboIXIsQsPlzxufOxp01A1/e6L/Tf+8B7s5hDbMqC0z6GQdCreeNHG4c6pJrzk72bvyxynpel4t5y7qLJAFqih5dLqW//F5Y10QWfU/iY1hyx8K9HyNaLqcTbQg/DH9eNeWRhqCPGGKtzAhkOa4tD/0BpK8Uw5c7aLIJlVsNeSDWcDYLGvNVyhcWhC3T9xs6qpOM/BtREy3reohNhnp/V8o8/X5gd0LDJrG6IdG1Sv4dpPjhQbdH6UOIXNWF/dJ6h/L7XGgu8Gm1GQ3JiIKmHwRjLG8XPS2Gtl8hQvJrSqJSWwxw0tMMAXKdz+LnUHOfM78x09L1d70g562WD4J4pYXf7qTDzA9FL5UrJRZN4u9/Fgz6J5P2PrEKsQanXAktpoHtTDTF6m1/fLtQErzgjtVDR2QrDY65QpLYS0yw80mKeMrUlDgCUGlMXkCepuuFd6Vy12SKVOlAGg7yD0Td4aNu8+x3ksXZLzfp0QcDW+2MnkGgZahDANBdh2ONCbMSRGJDzU4YJa1ZXghuffRwdJpujAlRlaN8G7U1yBDmcI72dCVOLNnG3txUyTj448sB/RixMLCfXvqFj1pyIBCedTKJMjiAi+W+2fTU4jjJeTK6OAB817A1vaLBVWjHzuAaf8i5n3oDocL3og+SAo9QkWm26PVEsmlpUzlrh0O3+R5JELxt8WclF+kBqFoV+xmoruhdCDKXq58aIzstxut4SsuWSMG9I/ScNDh4YElY0PUWFBX5F6lFQKN1MLIoHPCajODAFQcigBZUvUonqo9eJjzKzl6KQaI+aF5hJrDtzJeIXb69apwcTrCb9EEpQdZ0OdAcg1frfOnkRpO9viZvQcbhjK53iTSi6oJDnhg6ZbFp5bQTW15FLQEXbLexEInZqiftsCb/StBXMdevz6wixvNA3t3FZynp8/NMzJdnzsL9+gqse3ZCTJMEuNfP2cTzONduzPs3EqgjBxxDJaryjm7A/MrP4VsjCuqJDuA7pN1ASRIjXlI+HuCNu7MnPss/iiuco8Z8Vdp2fqrLz4+rja19dWen/okCXl/C2jW5aTv5rGszo0R/w4RYgl2Vo+4IlpChzYc37ZWX1oFKKVrPNOyPIW7T5so9NgordUqL6f7efP/hfFMSPnsPxVKpWw05FMiDk0r772PFRO1FPCZmRnYhM0bJUXClwx+EW7j2560161yZwPGjeuLIkj/48R9xL6N2kzR1bLPMLWs/LepYv4tDsf/2t+9xZNRwMMPF8wVPFKOgRcExUj+yrEt9oEpAJXLf2ryNT2Hl4z7MFn1+oYAw==", + "page_age": "1 month ago", + "title": "The next evolution of the Agents SDK | OpenAI", + "type": "web_search_result", + "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + }, + { + "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDI7mMDzu6EuE1sfZQRoMggo9OxaALayAnV3YIjALxrR8LDmldK71rzzZuCGQ/5DUxJ0pd+CgXiQTQn5wJKnSh6B+wKVsxF7zRM+RdzYq6x0koKeTQ8sTLL8HELygBAjAF5aItnE+6X2V8D1neYAczEW+g4zx3iYpyHng3rQCBkdbE409z4byO2f+W2+WBSUK/k1Rv5Q5UNxLzYqTqUXydT6TnB/lHADY0dPBdG5Nj/shYvuszWLq+fbEz73XuNkpUctQc0SiQRexiztplojR+LG/Nd3Ek8Fhnqa0zuZ7j+ZqpdM3zfcJYXQI6P0V1HqRkowcHRhROURswiPq+k4dqi70K60QOP+DQV8WON37q2rwOvOR3iMnpk6sNZUBVAkbk7cPvQUSzP6xjMzDvubpOyenlPOK8OZuIDyhVF0M/f7cnF/b2oDM0mDfkLFBYI1SMkTe7ezrG5uHDDI0nNQnN2fIGq6/VRCLFoA9EPat60K5ia8YFZX1++SPk+cyk/9SDuJxOvgl1xaQRGtXuLpD6RAJO6O1weJbSUD9sONfQznILkd9FWJ0WshVeuTTvKkFxWkThbsBfSTDO8OjWhr8JVAGsC7Jhq03uX4S9jym7bp2Paxs5/aIKUzSRPYkrJ+ObQ3K18xvYeZiPwheQBswrWTfPo8e8ZWabzZj5kRKt5qkPRdD+//792VagiY5e/cEb997DU5zmHcICw4enfAyj69dxhSYRCq0zuvf8a95UrYrv1zH8FjdUeoluaZtsrkOZDuykl5MIaxq89X9bIyjDBWVerFBFXIJixsQUyV1ZrRsUNj+mGLJB2a1K7L69njQ+1n11yB9PWS4IvJfUFZt/ebRoQOb6VXMNykzUkEzQVaQ9MFiCt2QHbCPqsl+tducv/TIfm1SlVao/9M9K0+nqI6V6GqXgxW2yzEbrpBl3UQG1QPe0ThJmwprfdfU9pItNSNSfiSaiyFYR7XG3INR7+7FHhE1+x2h4NRCn+6uA4nogaCqeM/ZID5SKp0BbmRiWJHNwP1f8e/NSs/i9oBZWhlUXjIxUnlW9zAeBFdBF/7GWixbXWQXzHyqa0237oTdQqFrBKsEE4fvky/w3lmOBDl/SNPFIoUvcx5dMxw9o0cZTekBaJ5EqdOBAMhCIrHd4/c34G7pf0Urgi0OxbQLarXzU9QpyR+GeLZmoEeGWIaeWMZMY2KP0pDI9mbtAwSCkR8j/l2nXanN7VxVfF6Ysqih6yvvArKhqKh4+lCAaKy2u9bgVgrWT8ufQiUrqtZPze3IZ5Uc26fMJHcRNvQ1uTreD+LArMMuMUtzQjIsmhdvSHftqrOrLTvbHO2cQDGy2tULZv11ROliRRj2KPtOOxSpG0j8kJtmOiqFRv36XPj3wk34AaOdDYosxGez3k+RcxGYgUYakEdi+YybnUnDLbBH3ltoGGkOhiLQF2PuKeEw72IHsicrgD3YS/lEymiCJQqJoXtZjBedp6yqgI54cktrMe664FVAYLURHeIyOX2vhnHe6iB1xta10fARoD55mRM2v5fkN2sad7drM26tuqycHvfwsoUe89SoCQBb5IO+LVYGvBqnA+MaFR7hJp1I8z8bDUDX3MnRxb89c/Pt/UPFpN5r0FxU9e52rJIo4l92x6ysqSSsixEa/HAtNSc+JzyFlcLlPmOQY0MsZ0HrdMIgDh4uVaicVuzBazSek38YvL4hY/pZnDb0NlPrWOwE9CcUlFvnCIvZbsrjLfrwZCVYpfFTOdQV6WReInjpBzrg/WgrTjK8O6HwW37gm8mkSCJ15z3E+f7BvoC2kwgz+vpgZNLVUa5AwoNv81IERrj4qRXhLUtJFEw7l8uiYYKM2uXFq9AnGbaM7/neBjAnS5+LMVhvJ9iK6XMHWHk+9rifQ+ttWUF1eZ5RnwdvIr5WrIcCT6Fs4olJkaZB6nrk8vvb9yrkbKooFvl7w6U9MGmdDQ3RMVqI7gw+MPPqWIhKswW5Z5dY8er2nr5BxT+ERtBvENsfqae6YBhrsGlRKGzrb1CG8YHT6/8Q8zam2GpGeMazOI8neDuafbAsQvOhp9LBSr2evnZLdrDhGkCEvuLssSWiVIk7fcT7To9EqXvFu6h3l7L/HqirgRu3Q2cLOSyQvilljQ8EawKNm1jEbQqRSP0ag04xSOLVlnTY73rNXG5pB/hgLvzu166VAAlOkoPXMtGMV2n2n8YBUof9iYud0O5KGY6GrdiUbtqyRL2+xRXjjqa0MoTxSri5fo064wTHSkl22u5qn6OPocAizwv4mKmyyD9e5KtnVC3X7Ws4xjWEYDX0S7yd3AErP58lP3GvCt5mNxHYhawoQHRDv8yH4XYPmKZ+wuZgdtu110MwCsYHU9w4sB18bhAg4NYBXxQmLWLvp3/gZac4Gt3wpBsFhXc+MTstM4kUiJwFc9b/wNY8bBpkFucUgZfcC7D/Gq/Zddzjb5QCB46e5ceQkMw4D4J8MVw366akOwH+ZZl2dypzxhpsFx7mzW4XBTFqhkAxVzAKNbzKpvgOHX909UiNj9WNfcB3pHu7hFliVZhh+8sH0K+0BXxGoKxnwVXQLXQrGXgGa7MPLxTEuLyHV54Jlrtb/9yizhdjDW/u4/TMhVzcKeUJMGGm4+sQxrlwyzZVLjRS8c9l+7u2fF1G/MXaz+PIQizeC8DTZXky/kdhXOsNlRf4+oqPTmNvBuNWDHkih9cvdblNmXBdg2gJgGCOCS1se530aCOUBe/NiDe4QuNywu5MfSuCHCXs24SEKrNCObly5TopWVYCVoGK6Wml75t1Uo47cybwL7MSJ2MHZuxUWh+XWI+Gmb8ZWrwbbkt6wCPXyQddqEW8fzHQwGS/HrxQYE0kEiBa6pE6bK5zcxMQD5D9Um2jIKts/g2g/gN8XpdYAiNKaW/edH9VoemuVCtmksO7vL2DNUzO+yFvW1dbhszclvayrCJf34DscKgqwI63iPc8J4GqA4UeuDE3WdtOXTQolT6/gaxYgQioQwOR1cZsPnDiWoszBUbCLfOFj3fPuxtXoRVbprh5sRhTUrGxHUmMwj0GnE3Gte/6bhtmIWRkSBDF7KuylaXCRuLs0VTNHPqhzwofvBjvP5GomS2QhSnCiDqUoD0gNhnm/j2h2uejaCaXP0gie9pi4xLLW15xyUIKwO4+yP5htC958+5MAvDV30c+K9iMLUMw1gtnQraF09w/RT5l2Qeda1sCI/oCeLOeQhmzAh6FZVBbx7xUJm9kx9t+8Sz/Qp1KGlOhNLBABoKnF2XqLtSPkV+GzCDoLY7Hr4+smN2oKFIBP5RifseM1wmdNI6qfw9b0CnNQvIDlMZC3r/Lze5afaStgF+U6p3D/ZyDu5nHKXz5SsOmLdN3ig4kvlKteX9EUld0w7aCdhWvio+oz4gSwXQRwrCNz5Z5jlDPXaOwoxIe5NIrBwSZ7RYwOprCUBcz+n7lMI9dXjwQxtCrXzCcQ95YlSZSBVEYjjFk65W43AJN0/VzsUVUtSBUIilRnoNikNIccS3+MQr2uMv0NVXydNCfIu21EKVyutu9uDLsDGyiWHoG+Z+peQlkkAhOrmVAmi0KNZ7MbhHhJn/N9sn5+M1RtdaWl6toDtw6V9QlExa5/xyMFDiah7Il5JXCYVv+UUz42fxBZ//CGI2PTGw11WLFbgUuMYcUC3Q7/Jsy/OYNTD2izHu7eZiJD5XfAX3sWygyHN4GP7XpeOcJaBlII/m45j4+AAowY2vg6dIcPydgzEEVpLn8zzYXghzlJv961M8EevGj1/a6ypaNT2i/K3MN/Dm7JsazuktEprYfv4pLiza2Kj1S7YNWu4dMkONQHNp1GB5Z9ysMEDHcDLdNsusxOj9Zqbu5dzhdGYmVXZ319j2S7HNK1pZskvZnUjWn9ZcvDh5K83DglFOgiWBPA7CyHpJXgTFG5BpQ1y7uHXbzMuI1kxVOt4I6uPanQscXAnWamH0MSYXWxZg6SBq0RHv3LVHNNAB2w3zn3qmY8lPXzfJ+qt73UP9dTl8Gf1nWSr5z/FVYDOBC0BjiNawIuq27qBnE+hOkoGltoyXxvFRzm3Sx8x3u6SMnSveeCoka4o6dqWOLk1zYlq8WL9ffTimdWKx0yC5tq3hdXngFdNi/ig4XT8b5WrJz1ETh54v5Vw732Llsa1tJ1pJXPGrJLhsp8SANklxgazcuKO9OQIH+UyY1KEdcAkW4dp2GDIl9Sx9Cic3kMEg1XwyjySCpxqBKS9V1ih+xktQM459jNe4w3HtxEBoF/X1g+sf47przNJEX69AB1STwThqM071oJTw/Iuft3rDG+XgFx9N3no974jTdSzjTDsSqJR5w2LY+emVDiHT8wUqZ2KJKZ+Q17r4J63sRpD/7t+r2/NPSiTEshUNwaVhj1CsCjx6Rd5uR1q8JyUZx5bGhvUG4pWlwjWY4OBDiAcU7keI/WdM+TdpzOEWiNyiNpIAgtg53Qsf4eZ0vQHjvchcONOTFUp+no8fNc9ObSbyzqsUON/YqDJ5ew+UxMgXrbZrgO8MmetbbRbg+xx49eJ4AHmoyGkVOPbXRAAHgjQ4+ZtsXZrgYIsYv6EXm+Ci7Yf0DAeh+caPJK6NUPI9fNeXX82ohTn6gpNGx71tSyNUcfCPm+JFsir/vyXg3AhhyAy+RfylVTU9jVDojjgwFXeSBj8Bml/aGsxcbyG79ym13cdM17U4U74vresrkMW+PKhvtoqEWCXqQNiueUwneKyKTbSFuuv+Hm+kU/0A4c0/cV96tpejUSm1kJILdMDadKWoup/cw4bjp+iZ4Os+JVQBA27R3Lq8TTWlYwSha/USDOSCmCaQMT52t0qXevUlEVf5SehEpS9wi17pzRFFKS+bN63PcXc222aHL0mE6iEipldrCyxBcS3jDkomnJppOR3lL80wxUcXcf4yd61AngGCGqR1INESvYikTnV1DEv7/G+qChzN4gt+5Hy2LrgPBzDLivszkw3C/1A/SIwvtnUnyhtk3v51yA95wsp/EDAii/o1ki8jrs2yCcXnxSA8VjfKAGgSHwT6SBvypoKNiOVvgCAxaR+UjQULlgFPTV0/euHrsCtjJAe7dIelhQcPuDnc+ap3KKnReBjCa+ZnkUan4IotZwjQ2+98iZgwYolNM+gqruYpG8a2njZuF4q/ojOP5VdP8/ZgNJa9S9GUFO9kkjE0YAw==", + "page_age": "April 8, 2026", + "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + "type": "web_search_result", + "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + }, + { + "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOsx5Tp5K7G9GIvHGhoMy6N7ZkBLJThQR0ruIjDsFF90FUjP7sQFJ4SjI/D6vnTbK0ZLbFaTusrK8rN4fEgISxzDpMf9rOPTxzHmmHsqhxGpUAjR/oWzZxmvaC8lkJRod/1AW82W8Su45qTLo+bmNIgNo7xlwwhhPkGkkc/u37uU6FJXa7yz6464z3DtiVAbO7ICKjLfCfCi4o9koVjIm3Qjuy4lqR2QztpNJyx9dWxal9l3OGbmyqCPqsenevRI+tAYtJbGs2RKr3gJSw6wXaSKvtVwBm4UmRxTg4lcLFwlZbJrdCeics109gQMYNAeZCAOY71z05yUSb002fXOzmC+G4jv+gJpKeDdP+Gn/UZvXMCbQYi3lu7oVd+tyc8XSTwR+HkdUY7OqxhWdBlJZpRRtS6zj3ukSqsZ9o2oOz9OtcwTbP4BXebZ1+d/XbUOsYbQmxg6/aHrX7XrWzRYYhGqH3an4duSCEz+mNiENXb605+Qm/aoKgfVV4sHSE57mEbX4s8j4pSNE+GU6G68+N4pG+KaEouAGZZ/SqXXfAZ51sn6ER8IOPSrllC9jBX7n0rKdFsUWUQNmqc0Sz/qYKdwsVh9EOCoUDua8ptH/1SV7hTQxu8Ehg5TKx6ClaExvVtKCiEoJhlO6S79HTyOPLcEu1Mig4lTUvPsXfG2vgYz7QIjCRnDtvgJ7/oJvmPpXF5BjUSNEvioXckFAK306HOIfbumCpmwrU6Mr7TGWdcuIUgUv3YYqBcJSXorAzI+ZvVmO7JVPkajTjKrxuyPDiE1qCRL+7v75OCZ/Il09ZeE9f6WwxDklHexBoSc6aResI2HaH1PO2emPYdcZZPUm85ONqzjh4IRboa3aZXxQNJkldjZ0drXd3nw03t48vtN1tAFK6X3C3pdIP8r9jiN/wtD9r6hDYrZoD+OUbgUuH+tciDr5sb5yXFQaPLCtvj9BXu+OX5aQTxjBM6lIfqxo8zfBMbvbti07eMnYudrkOyuqqUzujdVcWQjdjs6a5wxg9M+uuVoxBGmebk5p+D5gMGKbMQnqwGaeOdc9l8SU97DDsNCt6JnMy93o0ISiuLPcXSOCp5ZKk02+AafidbdhPbYKn9gh/UE7lDzIRJI+kLQvqYIAu7PeBqsa+Po1chyaSypC0JEkMtIkd1TE5KAl03yAHEkfefRB0JG1WnVN21p+eMpjYp55l5NSxOWgkSkU7kzw/h/nij4JlFpeO+U4U8E8KdZZWNOvjGQHs8FCTZ5GgQ/vftA1LlLLWL3bmmgMcHuq7tlH9Uxh1YJ2APWzU4IRC03oK1Q8BbL4AY3zs+EIHW7Ny42oyklRBsOi0B9N0TNCkZO1OSZVkgh3WzIxFVf5y2J7Aeim7HoZo2aqGgEe83NH/hWgccDEbZ9fW1kHf7BcsueJpPTt2/tj2cXt16ZHrZOTPid8R/lxBjmW4MZhv7I8ZhCup+RJIQlzXHsIJRvrl/06sDiunl8qpZl0l9fmb3hhoxbNyvSe260KRPZ6rzTBOLsaIqLWwCsrq7g2S6Iu1dlyGDgDdNqSFaKdTwroAJg4EeMCBWIroBLiEEoxhCILEhitrQI5887ivm1HvlaZOgwTGA4gLuzTfehwXUHmBXc/ouICWefQ8rK0u0il4a7IpVTdo4NZ4yObO2MytLtEuFD4AvK2Zdrwy4Y0Hn+6WMD/EKE/RmvFher7yplT5JUyjuL8TxNmaCXiHG8tEewy0TJOhHKfWy+iyfAzlfr619RoWMR+RaupbmQAnzCE95yYicGC8sJFrMsOEy9X63IbTerNe8ZRzoTOvczLSSz50+y+AiHcUolJuwVynWi59VOuqM/UHBdMV/LM538uYC78xIB8wS3056NFLjc7NdROBG6SpNKIefeufTdd/tKEBd7sxQWoM1MRypm4fgQl9G0GZgNca7YiUIGS8XtlvEeEzpj0t7NYXV6FbtP1DcG3O5FD6a3bkXP846DO0I1Db17NGGsBLn8UXMLM88P4C+l9t3LBHlvcSSYvjv9Wm3DvqXoGoJlaqUeRnqiByQikqx6sUkoZk9LKcpZcvwPQv9+aircSVRo7DT0nTjrsKtOZioHyi27YvdfcH1lU5V5ID8xd/pFSqK2kmaRfWxz4Lwh91mt7siT6ZXpPdOs5DRtelK/I90nBys4MYvRgrqCdTV1nHS/1D71gdsmgCCQZFTGSABRW4hi19V+xe420IoTpIqeFsHwswhDGa7OYJee9l3zaDganbUUWGQFnXaJf++sBHo50HBhi4NpX4+loI9dK+ZLUvEr3JsvEG8gQAksE1+T6FjABt70hEOoj4ByNnwoZhcm5Gv8XI5YSxRD3hnnXp1VSZbBwAArghMGzmf9yEoCG5dRC0aCp8X3BAqZL/rEcNpvietZhEuRMAWi1mHEuCv2sigC9v0EsuOUI0eR3ZHTiNBL/Nn6JLeOVTz2PnkhE2R8xsy5fXRS+pdV8BpcCmInLzB7k2ir8dYkrU8Lm0qasI5P5+EQScPRySRTQYnZmf5vqD4i/WGXovrvQs0L2YBDRaD+dGnZdLlIrUtiXP5+yeXzLgDvxpYEY73Zu+0oVyFUSe5q0khRkV1gc2gabV8GDaoXulpRXUvB+28pn7VTEGh00tSJ9tl9YaKMKYrI0cAfr2jn7WQ0VvJSA1sExT89M9HivnO0PWH23qRNYp6C9mbZKQKdEUen6TCwbeocWn8mojSR27WwUPDD4xtOWF2ugUXIOD2OVftXMDgBNTssVEa/dtPp0GJw++0sybQZHawsWMWd31GtlIa9ytSYo5E2Om8uLwFmxvqhKscAV0Md1tF2RFOx8+VJAdah0YeISiSjh9cEn3j9x9O5P4mVazy+/9o9AmBdl9rZEZGtz5QRivwECvARopg5J+z1oo/IAKpRtCu27hVzuvGahv8O0+HH2xUdGjqjkgoFU8me38oxP+/huqjXdcwaBn8r5+2k3HmZODm3B/9Gi6kj8BF6DDsQYhZPdTuw3O4Ye5oACX0WYZKEIxgD", + "page_age": "4 days ago", + "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + "type": "web_search_result", + "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + }, + { + "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAhDgusGthRwQ8gEaBoMZsaGfUAPQuC7mSXgIjClE7L0D6mlu69L1BaEeODnCXVcg0QugJ8eEpHtka+IXOLsFlTI7CbRCO2Q6lw7q58qjSDiF2QiANQPGof+sQVgoScXW2jQO4PgnDyRxEoRca5LyJY6DP8LVP5dZfyFuUTMSPb0QhMRZste5Hv6hJtoONNmcptRuWEpeWHy6BRxDC/E7YWoHn1eu/hdKrwO9VoncRrVAD6NNSOGkqc+rvjE4XNV+++1ZSL66sWqtUmi3P+ukLYY8Q+B3ME86x+XJC5jGmeHwVSz545X1iJ+nH4eXnsd2w+Q4OL56VGtvzZ0JC6sMJ0zY9VQvop9GSJGODdvIrqykgLAN09z6qbphbDnCjqP79UDn14EYBA7/Mp+p5aCAan9S1wFzeV1DLZZhrP62cI96TB5HTvSt+UttkVVerlTTTEe3AYZoYxGMxIMR4F2LHLbySeCCvIp/jYnj36tVRR0IzTsLGF2ijnGzS+wSkRg5qeirxMz6usrtMYgmVWaqTkpuFBbhC9Blh+M6gRQD/YBqvX1j3hvik2pJ+wg0gaMYBHjRtGY7sI+cTUltMSdP+ShtBMc653jw59fVhhm9sV2GQuF3rzVo48C6gWCElc/twGAhNAp0DrjvVgZuu8DL7ejDxTuK95cMjBWPhLmGh2Kj3SUbmLPDPv3rffWMGeX8mBq65ks85fOZkx53JBTwnUGJ8ONsOrqhP5+gfkscv7mvtylqbie/3DZJZ30N6U77xrmLjTiQ6rf6Z0bYQHIiYIZeSwm5OGq6hHbIeiZpgaPXgwEgc5Us1v8ZO39QV4Jxegv6BDNaPYvVjAybS++8VDao8lPfJfB9fNUEnR82qxPU+A4GhdQYxcyYAfq+Uot4vEffUxXrSkTHl1LxF7W0IqHFvecv9Qu3t1/spXqNpUVGJLGCfUbu9Dy0vDLEJLvWD6VY2AcUiTN/bbUIab+5uHLjTm9qJ3inwWYpfoxQaDoMsKDFjTGJEBFff6ky4PeS6BPUAOg6FFi/XYAbzcre8IzyaJeSV3MRbniC9GYaeLYqMwsn5M3ZK6T3vb5hKROlEJLlTEyQw2VjVfjrwjNWcoJatISqBtZP3fwXzg5o3xCNlQA6O17pOEWbPd0x5jecsHF9xn7VlYGj34SNmJFhbN37LJ/4DfrzyON70WFGx6ZDfUwgcrZHMS5yBLsW7uWd9QcRo9gmU3yZZlCZlV4mrhthnF7Y2rTDNa46u+As6WlQcW8d/HOEg2hlEFhwh2POsCve/GJmdqdrzGSVXz6PHqg2U3fps5TGX0PVII+rajDaw9/tuYUT5EBmUEq0z4Su6rrHMQkn1novlPapFVslvlIW1ipdPBCkf9lL87oHhcsg9DK49fD8AMfK3TUe8/LD0gQQHNifUh4RoFa27NNIjvzxfArbssF4luVSDVLwNayGvMAH9MMDWAeZc6v/TMzg0Hp1L6iJItoM9b9ZO51ptV4KUiukkMSJezFc1ZKUgL3NSXduHKngMUpGdKkQjR+TtvBBlED1m/3sTXlSHlck8mY3C3JujI3yFfQgdNXTD/zww7UFl6eF73/U5QvC5WHDec4I5Tj7mUDLtmQC2mnDrGuxIJVHmz7gTuWu+q6ixAiOHuHw1wGfjd/GjLCBePwF1Umbdlp+cg2b99nLsGWdqgV0dfMGd31G0SJI/WqCMty9L/uOCILRWz2zWa4hRc0aNVZIK3x37AX7EL+3v6oL/69ZV/EpVaXnbOSmuRpwE9U+Kyp31CGiVNYBPjLjvJ7D/5d26gEdgSuJl9uNbDYEBd4Quhnufz+Ue3hGOdHKt/34KmUQEm/qJUTu4/Zw7r1dQ6Co93CUA6XU1ZsNT5LYzV8LLZAT40CDVgm54M/NcmWAahX9XrYc5CBimG3EfeI94lA1zT/NuqzYbx4bQoCNtvSIDLT7F3EvtMMJxsc+zsUWiZYIhQSXA+3CIiX0xzZSIAtKUE9Jgpk1HLMXCS5UV9VRKshkofam9EaeODhRaFzL7+HCue+BC8bqIq+2R/4IyXPihr/5nnfCryIXfhO37EXJx/VsXCBUQsgeWJZjpRdsHCpK3nDkYjpi8g3DfjHXiOSWg9qO4hVG82FUU9oNRUOjiINd6CqMLBfOPPSm8Dipa3sQ9EYpJMDjDTtmQR6b9yfBLI4bUlUaOWHedSDdFSE4/1L2mn2xbWlBCfOFoE2ls/e5fhoGm6dThZ1SFFll1oZq6aI/pdxAlUmbklcf3Shh4j3fLmOzxncC6zr/tLDWOGln2jgx64EeN7WF52ErzN8q//I3ADpkV0qAk5yPL5YX4FkOeSsyViqZJhUSy6wIuIPKVuHb2vONeMAhwTJbfGAUfDIW7VgUc8MDV9QBH64Jh6jkN3rLdOtMb3nm8X4lCg9xpRNxf6B2ZdzNdYj/3Ni09iUaQacQTcmgr/aeXI9V4lUnt+eeHzKX8rGUhvr85rkbiDB5CcPulxXnVQQRlfRzqNZANjCCNDcWn89iSMILf8uc6BqkO19gCKK2aPdQmdPCng8qtJvr/LH35LYr6Hxn3AvUHESwYRPjKpMGNkpFQbYy4BvTjMAijFw9BhbMUvkeYEkX35wIvLH5ridBoZawP6WxrlbJ7WKr0jFwCBgS6kbvvRR6+Ixhxhk8VReoSdFUv3bNSjSDvP/bEQkM5Jutxl1oHU8S0ISSezER6IU45UpcuXMKqOVLf1Lh0TUThZc9FArrcJWvnYaSYUwr4pggqJs64NnOInq8HSa0hWsfouI7jxff7NN9jta8ss0pzL3zeKkLIbTCOFNfTubIKuxDxONs9nOQkdIofvD4jLJolisfGm2zNKMKTO3TX4/xhfW9ceMuEyfcP6xrPlOmdhpdm/xwuP9WM0yp7xNw/5H77MznKJ5la5gmB0oJl9lGQMkb7UYWcGshxrjXg6voJl3aKEwh5s3n4+C62+AP5IneBvbpyAJ89zLIzuR0xCe6QYE4IBnRfqg+GDHnBNHeSFGsp5ygT1T8t1WmwP/mCaZM1XJ50dznsvgxnwgAC0LjNYxRmPUZx+3R33yIxtymVMHcaHJpoPtobqE6kVV/LprICwF5hj3Vg8CZ6gzT5ln0JFZUyzUdA4FHoZ3+twLQ6wJzVED09ybTYmivw1ci0siMl5ronuNL06cNzQ1ITHIwKbOS1d9DnhnMnX7xzmEIUE2vnlBbu6LusaczySR1uC4AvcScLvPjGs/m3HZhVB/cXsDuJ1Pk0nhOJK/dEZ8XzDsHjemTUahqZbvtGKSBrbDjjYbYGT/7WTY6D4UgJEr7nV3t5MNMYJsfT88hCokUxJphOHL1Ezs4uNso6NNoiCCjTYKvNKW2drae+TRo6vAgOIxH1jw6l1Kh5J8tofJ70iqgssCUPKA1EJ1s+RWdfgYyA/8oYoc31GPuudkEiauaagr20Y4AjojqbwT9VzxAqOVDqFB9gha/D1bcl9uOjeWqPCy5bZCxeua+Krz/3+s8KYv8hL2CVvNmey5otWqXpsNRBPnnSMt13nowXXAvKxHmn8AYh8zJPEnW315rFsSwD9gAKitGlkVqLzF5DNk3KkY+n+JVheQKhqAmR8MWwwIhCUsPHt+eG3mkujwihBTuQWVn5Tkb8X1Xeooys0dh6w+Szc+sZzPZeGgIL5xz/SWHiZxEfEpA3TJbwDiD4uaJNzsxsiGbkhD3heuiROwEyg/q1En3Vih4OQ431se5J+jvEsO0c2Mn0kUkZmqEW3mQBHVnUwBgYVC/Zi/ib8YQ8q8ta/ULTlQapwaVozshFF8T0QdMXnoIQz+1Z8rWILY500M0+C1tcd8Q5rJOYIgMlVNrrOzHR40uj+Ng0U5f56g7Gs3hNoGg+Vk8R82yUNfdGngS6Q+BoKkppGrcoWyggCmztYicb5vckykdRdXkxocNtpfUkmK7RxCXSS8q84HLhory/JIeBCR+KpuZSGf5S9p8o9qpUvP8Xati7Y30wjFLDHEUXfHiV0v/3T/MBGViTedcXXyVO7TSF3w5c2sKbQ7kGzQXbkCYUcdWxZgTOaI/FwUowC4SYQwChVn7qYbD5bvAxCfBe1ENJRy6hG7VYJIkZQ07Bam94cuGaL2ryxEwJy2+3P4aOwnumSmuPi/cQOBFHMjXRzAE2lbSvrTEXW/T5DCYOFwCbNRH3OSMwTmCsstHedVkqiQvy6fGRx3tZH46DV5VF1s6Ya8UYYwlK+mLB7B5jDKl4sngIYcmDou83TYCBOdQ4oKpfIwrAWybOahi7p/JoaUHXLpKH5hRgRRKOqk6gMpCR6PV1nO42L4TapP0Uo1qa7z/BuLyg53te2JCiGK12D8AsSohYLKJ5DZsPId0J4+oyiOg+gTbdx+cqNRvFerbm+Lb+3+jfBJwT19l6xSRpyiDG7jPr5QYNoR9q1K+qqdR8CE4IXWMwn8vQoVDTx9iaiW9MpyuKidSzNPAXdQwZfbgHYl8cEGhzO69WZ0KzTH5n4A7w7g4xzs2iQlbprxOhYU5ajNOccKN/d3+A3dXmJXsrZjZ6qkQq17UYgOhxo7SkbvNLBn2ZhLU+krM1jVZed20qAIYDWMxNY2OSP48/NYjY4wqaDeXYzIoIgnoGWWlu12W5FJ1bx8gErEAYBGr5lBIs9Bv6ST7Yhpee6BynoA01L6WpzOW4FrquoTCR2VuE0MpY1GJLuy+TzI4lsj+OZIvA1LmastW3UavIP4mL5X5lZ1vxfnAxjuX5LaD4SwBc1TezbHZ4t/jhoWGmqSn0MdIG7m3sEJo2Rc31bsaFi+gzksHKp+WyydIdMC6zlX8AGXG+78KbSJ+4gO6eAPhX42CZ/o4dvmwi7VINcCnh+cAesRX7U1hL2vXo0+wLv5HiCKO6KemrIoVzNnaAx6qFPAHgR4t1x3MwwMmTw9UWoVoY/JUtIPsfeo788u7q0rZPI7v9KByv8VNLsUPKegJcsbC3eqVyhJhfB6P2TKbzZJh2gaaIsLKv37ECBj/9bwzVRz46RigEEO3VWVxTZd+ikmiUHJtcapROsClRZBvKUzXESgt64gHILoX/Nm4VoqTOTnqNghHb+kLoXb+jlbKil+9c9ogNWUtdj96opHI1nE6rz8Y73eQZijKx0BeQ0TrOlCR0lLZ/94FeIYuj++gxdtTMzDBocyLph3xLdXSyGXBgG0UXI3IaBmSGSMJxUELf6lcN32kgqb0pbsEVOxdATPwMKAQuni44vc3PCRp7aoH1kBXPEMR90ynXXpPQUloMPS42O4P6cAJCw9WvC+3S2aJarWWjgBwiP9uYgtnUdEF4z4KFP+138jUaBwjCMsYshHdXnBY+zjNpUxufqkg3p9YA/GYQ8HKyxj63sOmc9SNuOzCuIg8+J5OgtfJ1GHMp5WBcRaGjmG3MbYGm7bfxouklPKL+jfaluOrm71s1exVZGe6xQs0tu+nwFjOmdBJyvpu+gY0v5z0qTibhFA9w8IACpDD6uEVO0DG04ES0w/RzkJCkyHQnCqphXusn8uLC1HZRBSytCREr/24+r/afcC9iaBI5EzCyZU7/jfMGby4aYMfJK1Y51R6nqUmL+stjT3PhgD", + "page_age": null, + "title": "AI SDK 5 - Vercel", + "type": "web_search_result", + "url": "https://vercel.com/blog/ai-sdk-5" + }, + { + "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA+bo5DECQ7RbnLGjRoMu/V2rXAsOajX63HXIjAnib7omq+dgSs2/3/GJneFB2vtjsOGs0cqD1py6dBQJhycEmg/XrQ2nb3qrnvk440qyg34BcSaIUyo5Umh5MLJr6w8LfiGMmjiSeYGGCiV+8N9DwtPuR/nTjT5xDBab/uarBOhmUvTps2l5pN+JWCJDsqn3mnMhPlssaTH0kNq/OdeUx0B/1DHfdO2j9QkNYVSIw2mpXexPOow+f6GLfpjxXSq69FVQZ7qoXRxktSXrubyrWF7v+/PKsuRJcip5X0uY5+ioFX6dDWmz4qPOcYL+XjUqMGaWwWggxUFGue2rG2szCeMBfo2jupRsAEeQBy0ppt2AnFjIQwHIn56fAfVB3VtGHQY8UMfD7FA323POnPg1JncaZlEPlarbAZLXwW9VoU2s6Xz+wtxkbSdR1RprOMcjfW6Bnrpm6sCGIlYmvRkJpKvqu68HauZHgOC6J6xZr0yoBP6PUoXvDXIjf6eYnjupPan3oOd2RsgKPHBTEgUFNBtM8uo3hnvnPrcjU4UVHvjcD9i0mh+WeCblHu25oCsImCVIt65nMHlB3I+imPP9oxphljgp6bEtRpkCXlWCNZ+HlT1SeH3ABBa4M89Jj5sygm8hSUXoVDP4URQQGt1G161Qd6q06T+wWPLhng64ECFtneLbOuO5ft5Rkc0FfG96zYzCKhk303NJcBxiVnvX9S58XRFDFVQCgztiDmnpCwppU15XBwUZtuOdvhgGWtiqBiWPnyGEVE9O7Hf+nZ0AHZ6PEXqdhiIJ4Sgy3RcyXS537cyxY0hzIFvjElQx9hHFsabnD5lHSk7bGpnGrKHguikArdLQjSLvyjX0qt+VO5mtmfPmvbCXdt4zM4paJMzuuaGu8sFornrq+OPZEuqBA72lALjYHBKDExGwBqw0YZgE3rITVJcwiYWuf3u7a1Y6cjSlsDy6r4kq7riC4bKLBLG3Lkmh0uOVrjLlvYFatsBSGmhOOTVb8hVQnU+/lbuzJUDJPjktXNjElYTFDfJARyCmeOQJsPJCSmCkk1CXjtOmF0oERg6v36rqMoawUeZ2z15lXfvGH30KrW7m5ss1XQ1l9dvnNCfvd96DAOa63YCP8HBQjLt78C/F7hDTtPvloQDIdAGJmtwRWCr1o7VYnNyhpP6j4yrRIlkkMq6DQTETS26+NdpjP/v09Glde5UrDN2V1V+wK0LK+TBIJLb6QB59GgMLStBhxwf1pMsOT2Rg7tkJTesfswcJQt67+3PFLy/FM46tSd712LLKM9u2nfcQ7M30uVVy0LXeZmROmdaNMiaIqE5G0WLynZAQlyKgF0Gyqofiq0adi22pVB5tYoRP32nHy1Lxae6OaXlo+Hb1P4pZldi2OtO52olOC6yurCajDpOWHzwN9ji7cMGWCyp5nWIK+tzZ0dHcHwrKjSH+2TK7acgVr5HuxM51GDUVhvXwUsgXAHwWxeFVgG9yDB04uxtqxhHtX9Oyc/R59T6XZYhxmcF/zxo6w6syY1svn9XckEfuIsrDDiFQrxV7Ns6w/KX4su2+AZM5uErdV53pZyW5mA6aUz1beEwRlKNSonRg8Y3ZG00B2KAmX91FR0B24pWp32BUz0rvf9RFwdjnIZz33MnErLlAZTLDRM9Gfk5Le1/4HW4nU0Zesrdwh6j0DGkgHPbi0aAbpkPT8GgxQ1GvpNR7q14o3N6/zkiX5YT1c7jpSfO/NHNDGNfd14/Sg7LKTd8cWorKmL7h7rXPnJIKcnvQIB9Qi211yhBCCdDbbvnMFkWFWXzkIuzyaZAZAoY0dJeXcIxBqR+6RytG7jYgKuoHOgMDuVoBJ5QnOKeI+qEWiKu9TLSpAUijwfK1UzEbix9HA5ZAu+hLPAg28Il1HqXOusilIYoU6N1IZaB86owGfFBxiTPeud+H/aVmlaHStBc48vjMJyBH7klapsZzx3nA/dVulbFREkX0PR0xpWb5+6qK8pwDlg+swo566y3OzwP3lrvzBklnvhWcqKzijUJismbu+9KhNJY9bGT/JvRwZc8P8dhVlpvEx7TyyYjqY29nvUddyhDlb44M2Hs8I/qy4Js0xwbGlI+mQCrzXlF15SrZozKW/Swwr4R1CgMM9TeY5nutgSHOxgMSlmq8yjJfvmuOtQeX7c7bqZ0b/OqwESltmA7mM2VWOJ8JagLfkNnfTKWL+GLEyC6EepZyFKbK6MShrKh3cb7j4t8pOvAm1fz1pB17nltbsSHe2soaQcMDY04+/UfYVOb5kNAWQOgCNseQi51/5GtJ2UXkNi3F6nc0Kqz6WP729ow6GXGqKD2cnwGtBl9VmhpWIBirRHTpVW5YZTphcWyRI+aQ5711ajDgY1bz3BSIe1nC8KHwi2JtPkoL5Z/KFiVGqYQhArBlxqvGAM=", + "page_age": "3 weeks ago", + "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + "type": "web_search_result", + "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + }, + { + "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPOtDoyTBsMCov59UhoMPbunuAMEYIXojwuQIjDJZ0FIl+rZ+0pOp0C9/n+pYtV1i4BKd0hQNPSuCDxBQ/QgIjdvHJhJ0/KYGjFNPhgqtRAqbZmw9eMNiHfH2JEQtnSgG5pzvo7S+2J5x/sqrX8bGQ1KIwJsRLxGiaHUxpVofrQ34HYra7TSUZOimRjoA0mpPjd/jPiTLjpqhBNaZqkHcnxYDpsuxeC7acZ+a/nSyGNCP4mY//eml/nsfwiB6qltseN5unYDKM0onfWk9D3cUU0k8SIOfDk+U803Md33D+e3QZz3Bb8O9XO1KC4s4JnypnYLaHatx3ZH8F6C70v/tJn+cj/qtQWOUOI+L3lbqVgFA/ykhzqkPojn1YWOtAjHTeidOvk8KKH7EMPv/DIfOJz360SF5WHxHOnYxe+aozj9ZU1hIyvsimuNao0Wx8iVGmlSAxtfTEoSDo2ohlQpELSlXraLXZWAYKJwt9KfZ8/CpauDEf55ioSwuplIxuaktAi2TBOYi7cs/GvdVfSYS8F6PAcF3gSaqNVxWp3sIDyW9hqzsHRP9YAf+vBVuEDbfkouEDIvpArRxhxWkN7QZ9tuPXIbG0QK+LpezjnjDhbLE59dAG8cq8gvPutkuIsz+XKnlO3C+uF1/BSExOJMYDHh9OacnrUUYATmkt6CblpTJswCuEuUecP0wAQ4QJV6wbk7d+kYjbKYTUPkwlX/kRoKTg0a0+bhy8UkHJfRMjNNXPsCHc2sLBslvlkSiwJNgfbzpVDZrDg0C1YY6xlfY4+z+cuhUVl81zsChcEFhzS+7nsNk2rs78utc+BffYgJcu8ZsA5c2AQrJ8ciDIJ6jnxscL5liSd3wMdrnSPZRGb69QvgR+rwamFZLZz9XrUGzuaDwS+VqbTThytVtAKJ00mU4wa5vl2pKOyqmc4BU+bebOPVOUdjfsxJh/ToSVsPafN4nS7LDWeALZHA8gd7N5TBEcJkUNlWI363S+PkqO62D6bnoWTn9c0Tg33CKxbXfKbmLimQVlruGHYtFFEl+U0iunu6UMUZ35+UAJ7ECXGlgcoeo6wB3qj3ckh6B46CQpTV6MzemmC/Q1N+NGgQ7MydHUHVVLTqEcMILaST9wAzoijIX/fxRLVg/7L++9aRug5G2wyx10+t4AIvk9Gf3OD9vTVTGjA+WVZ3Y4R52YVO6DlbouhqvppP2sdhdWx3KugpRi/zD7cPbLWt4Xx731hp2dB+f0v2x5tW4w+24U60zGEMZeljfmP0CkEEDEMjko3Z9K2fJn3rRV/kgHTrAUvmCE8GP2v7N/2+G0rjjn1M29zHSde9uMmmlbJ0/LpG5G+tvtjno+5drCcGx80M7cji4o3TK1ZHK4f4b4j0MCTRGKWQ7G67BLbo3+qTktFWPti4dn5PcfyHRSowq8l0wxLGR7DTRqBOS/Q1gQjl0nGHXCWw1eKaNHZC2ttftPVb9dBxf1Bdx7sXn56jcig2YsQrvI10AVyjowhCqCTfWm4p2QtZ6PqKFoJzdHj5oBCQbqcewD8VlUZDqBG61v3NXfWN/EFZTjQVseGU1CGWGcmoVd9pV83EaxjM6cDaDMBkWjcdgdyssWpu42q7iVr1+zIItQ1aJKSuQvjRlc35TtW54pLNM6uKxkd3m2NT7YLcIl3DeUqfMx3na1aw2cKTer51ekj12o8YPMSrfRtltN3rVHmtFNCNNv6WXrvc86O9A3dmW5ileszqV3GP/aA5VhtjvU+wQ6fkZStRMAxUsEVKtVzeS/Mgn9gT0oLlbIx8tcXLaFm7W4g+ASgUFG6Y6SYvPdHdf4WUscvR4LNl0QTmui7T7qvehchfRJk3hfbdxp2WmXiInKZwRr/ZoH6sWa32NtosURkR0YDmCIOjSfa2q95zhl9yy+qaMXMlbTfNBwW0eKWKV8W/rDOvSfqmvEDW3T9x4g3nNMVyPos5I/4J6Rwte+fxKdcbrvwH2N1P/KeYpBGW8JyJQL2ETHYqgyFEkUPVJnE/EVjwMHNYyikUkOk9EHjixk0cS16u0H3JJAwtieUz7Ci0TG2kzQMnULo21/I//snJxwEUpFX2RkUgcjburInDjeFS3vXY1Ew1a3Dixtob0QFI8NtVDMYNERAWH5DKSqkVLYghO7ABgERcqodtpXOwNBxRS7J5beRY5M69U8Do7ex2du3uiEYY+M/aPZrr1uvVb9xiVzt2g7TBDpgUyg+Kvy+kDRVevgggX8W3TBbSc2Ow/ESOhC/zKzMgam9+t/K5uqe9uCv63kqXECXfH4pxhwY2E/n8MfMn2o3HlV2m/2/XIUHLoo/YMvLi76NN6sO/4NrVzKVyptR19yZ/db0QVWkNYfGDx+mzUae968yUy+27wXWzLu+LHtFIQJfEJk/n3NEVZ8Y29kaQGvWz199X+BS2F7n30bd1suDU1O78+rGIsPa6LrahkECqgKOOpvhQ55bIVBBnQ8zFzphFPnoNlShPANjK3K0ifotjUeclyzT3DVFCGugVglRHVo73uxWUX/UbRVyj0r3ervXDiB5WC1xGlvPpQVtjAqSeiX88SYMXYT3MnaIk4zcbROtjuCoKUbPpaix54PDBjZFK7be7iGSdFfKiUMFx4N42pSzrJNM0N00DR1AlG25o9eHN85SH8Z9RF2P2HI5U9nyUf8Ij8pxZBCWqdexT1T9+eDd7VZAum9BgAq0j0ka2wXL8i6w9H1VGQOTJZnm5tzjWIX36KgZqgr/sT07a2I9FXZv/i8vVm/7NjfIskgp5dWudfDyMJGCI18etdOhC29r7XcB3lWYhPwaqynnZW2Wq6YBzWdmwE1Dw4p9ZJRW0okRfpIeyVq2YTTy82divlmT3bA6TlaxpKTiEP4UWCuCy/lnvJ9F0jlLm/Ig6lR45XKbuGAM=", + "page_age": "1 week ago", + "title": "ai - npm", + "type": "web_search_result", + "url": "https://www.npmjs.com/package/ai" + }, + { + "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPfDIJW1qbxKekFpQRoMg0iAQRQdFyNLsmQhIjBhm3VX9bIr6AykKY5axx4eIBZxrv7ldn7VIkWdRFUUZIuU3UYk08+Bcjv3erHpUxcqixeyUisUORxBWCE/vXzSjB3vSgIqz7Y2PFZKU6y9oiEo/tWD/H6i7HUs0tg6oLR3Uy46LwyZltsblz3A86cSYSeU2HAVU0PRqtkgP1dwyLMS4vkY9T64NpqLEbNmAWtOwQB3tbwRwYkGNC6pwv2nwhbUDVzUqMMQQSVKybETCmqJmRXlYfDg/DS7yl1RdBB5z8fBvfswEA0rmS1U8OmGwlqUpO/51CQotJOvUiC83mIczCdxoVGQynpFpZMXtDkmQbTr7IiUL16qHBHazKGKCtEKy4+7FG+JsP+LoIila13n4RVydyA6UCEdngauJ1OktmJrkQ48f1ZnXFhMlI9innFvGgYdjlNPd4EVu7IBD71dTDgQY1X2x1o+s2y3luOoy4HnMp+Ifrop2wKeHrCNDG7zXHmxkHnhJJu0xissBQLs6iUZ/FTRYPJb6mH7E0oEcaJZsbTjaNqqJMWO7qzsqQHjJlrGVVWDHMa4FSEZbpLcOsIBSK2mFC5+NK+48xBuDg4CKAGrf8vAOAKRaNiuTxw7XurmjYhC7qgoErnNPGkThQVh8z8H/Dq2j6orh9mpRTAoMmFVPzq49yWkhwUoufNM7+gRzKkVAi/XEx3A78l+1CFcHR5v3REiufKHQcJQAVkhk7ZZd4wUEu1S929y8aCRi7JlUeZ810qA6nnxXGvmmGA2KIm7S/VBOd4ATTADBQP5r1EqzRC99I/RoVJIr6A/NPIpste6mqDK72q2sQQ/C727iBCeh56+mFPq9yzWEig+dmK62N8+SWeDnrNCGHRk+KTM0Smp+oTn35J5gmONlQuyqUMh0+zfs9y7TXrpxQnkE/gCAlYCpFOU2wGqW20cQ+uD1elG15aCX5yaRIX+yzazVwFxV6JviUHSZXTWVWKddR3PgPZ+lApc/yqLaiTyBgL7JXpNI8gGRLhgOAZq+LGJFdcrn4EDRSygYZmH4itglammiA12s1zXrsxmUeoNEG9ECojlqWWjvnLTqrqL5aiYaP7MV7wGKGVjrliJfQGwBCVS7D+/gezhOxzs9hEtFknG15Z8JWlWE75pQ5jUM5Mt5+3m6cEEyqyoN3MyBjDd3+epwiPntVMpmBiCqCLNpsIpnVk5ruKvstPUMq+63EXUbJ3KNOawyT9DBtjYbmjZAQy9SrBWMVLHoQMzYJxwg5+9C8wIPdRIoHJT1clX5q0lOMl09IVCCG3EKzmgC8cXlB/ayUjLqtENvBIWd141tow7ylo0qEBinpig1leUuBwbk0CY8dJFFWpNtIAJG1qIl2ZBRoSFn1OThE9fUQzQN/qt1p3DHi35Ye1gLNRyFyAQQmxuodEY0tty5KTCmR5yj8pUpKDQ/bw4kHp5sjvWEDh8eSDOEMrD+vg8ezRLhi1lkLy6BJ6OUx4xFQM1iks9MayFXmeOcWD/ocIwcRlOi6QoaxsWlPCnVYj3TdYIBCxcYIdgtz3IiT/G1LBJzLUPCAOCkYohpGDiGbn+fAa/Z4LJ1NihJmZBz8i6LS1H6mN4TNFYtEYQIcp3j4XK4UVOBi4nmeIURbvyYoZpnfB3gXh/Am+nJxKXv/BMH/yKfNZTKawZuemsUrju//F7ofZkjjZ4MzbgzP5r+BA3ZA12K31wAPdF2GqwXaclzIb9AUCWT9Y11Am0yLwT4UZWPcgU0HYm/jLJARZry2fuwcZx4J0sIIh4yVU2zXipb1id8qkJZz3wBHii98pU1iRUpl6hnj/Xu/hY36YlLdymUQ5014tkio7+z0cKkrdGvrxGHvg7uB3DxvuWnhy9/OrgB2QQV8WOhqNL8vLXWzxwBJGVyRWt4q9RuY6S2omA1vH4BUC5XWTd5lICubWculAr7VUJnGcT0kUb4rd6C6dATyLDnGX33fZLauvaVVvOyzpy+YK8DF5+nNuYPnObISNHYpSdj/roz0fgenLWz1OtCW2uxag2LL4YGeKSmLVtuHglgSoKbw0gCDBnk6WSpyW529Pe3SLjc+zufyQvJfylQ5u6vzoEhafMDsnAZlyQP8WeupQK8uQF0ieyY/i9OXntLKmm2ehV3hpRK0rtOpX3z0YXLcCvpNIqf1R/yPWjHrNnWRu2hyJ+5QEFpTgXVBYUFrRITYZAfCIC7G7NnjzvY9wJUtqv2iH1+Z8Ynz24MBfUAGjESKdh3OFm4fJgNdimuApPMZieZVVt3x9+blLUvaBGi3hqzcgmCdFeA1G1mKjFi/CKqAMtWmVmpwGgtVyY6UPq8NpB30VhOYBCM+KAwKo4NIxFSrg5tYfWcuvCt3VOARFFxIBi2eIKoVGs8RTSv07rYbuxq0HTHpjaCQccuii7emHg2Wq+MWNg1RvYuXgeB5PnygKkNqI/51w7b08CXtyV8cL7xP9kjvO1P/z3SogFtSw4/c7PnIc2d08jDn10+U9QkIBjc4bEtH5sWTxdhifRzwZQ0ApyAHBGZ0+wMVFtUC7Ik1kBMN1NnX6Ly5K3S7PFRJozIdXGkFn7PM5YfmNFMEXT4QMVsglO9yPkZkp4CiWcrw9X0E7pEFjuPYYBKCAoHozLQOGK3acYw6EVW4iQWJATv4XNJdsUWgzcLh6yNjY8HUHa9iQKuVT250xXFlxumyDlKVgazCXBgIf6jxDx8MG+VdtOiI7xMKvTaSBi5pHDaYWERJ0JOOdMUeNpVpT+Okudpka6XqcICcrWx6SugQKsyY1Vr8yluUO1FeKhb2ucZ0Av8Xw+NKyp3etPa/TK0TU862s3RO0VQtXUsa6g/F9jxzuMDPx//nsVMB16qpOMAngOI8+KVUNsjM4a7S6AAtN0qSIC1l3tKQpLNGacbv+S/hP/pqJWipt2QqRWa0Wfo5Ql3fGw59fAEhEuHdASuuA16LOIr1HUGcAtmAxgfDZjT8DBCQqzjBSnTF/7C52xciCjzYO9gXSdr7JjaKKiR64nsBxNP6pEwOqyjYx795lt+YTGHXhDl/Pe6T8QJPD+o5Ej46evnzuKQY1mLE+WfYayGesOmHLpAzrodAtTC2PFYsumeWwn1cyJTNrCgPwvxfW5fEinBneK0P0n984ljRVSGiih4bfsE2Cul8sFwxOFFTP6reT4HMvVphFAbVXgEKxqy7uMg0WyxTYAdC8Plf8uUgPYYgHJvqRSqZ4ZaOr2WNuIi1PHCXL4EX+j/cVqmtsPZ3Ia6mr9DRIc1O8gHGP/BIhKP/y8EZ58ny2v4ns9JCYfscYzmqLht/ZnWHEw+ROQFlQKrrp7WVf2uMbaBdqMCy2DyMea1+fZC+bgVbHBEm7aIAl7BhOmtHFPzgxcZv8Fra2DCNATHZneVf5CGMHXfgK8uZG+CVp/hPB13VZA3jsoaoY9BkHfALrfbO1HTpOJH49EKOczWoyspRuIWlMiHxNz7/+QnBOaXI+xd9/3JOOh8kUIa07ZrXmGEmcdbALwl8Be8FXXJ8F7KQvGkWhhZyj9V41UBRezf510bGodUXiijNYhCgovFWSlD+9LtVzKi2qRW7m+sdSnfWa66eft2oABlkGO7DfcbUuUclWXRZqzxvbLe9vRBY+ZzN73E9JUFwFKTd57ixV5Q69/JKTJhu2PFGsXNlUBy3944VBNqY6jNUrnPM39u0nLhGDdt5/LXN9HRYXOet+Nz/RKgFjIf99J8vHeFYnPfsua1XyqwrjTs0dEb68ri2F+9y608f1PmKtEuPbm7/Ofn3wfnZRh+iJfKaOCuP53yTVPsEPcXa83IaAaH03EuDPqQhfD7batO343Py8UMgRzSxBOCM9DDc5wqnY/wlIrz31vrpRlHwmMx+FXie0PR1IrsAVAtKWBthwfIQgVJd7EzZTcjyn6tJPiH+C3Upx2OE3YhxN6czkZ39wrt5yLl+KTRfDUnnkb6pXL5P9NaFQ/JODOSEE0iKFVVH7lm7k4jXEL2kiLbjmiVIqtAdCvJsOz9XpaU+Mqzg6qGD0YAw==", + "page_age": "1 week ago", + "title": "Google Cloud release notes | Google Cloud Documentation", + "type": "web_search_result", + "url": "https://docs.cloud.google.com/release-notes" + } + ], + "tool_use_id": "srvtoolu_01Ra46kxprvPzeWyWomhLzvK", + "type": "web_search_tool_result" + }, + { + "citations": [ + { + "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCthnxqf0NqGDYOyMhoMBTmtvdkdih3VOuzSIjAHJR7oA3zyLnJ+mcRNAaPAHZh36NC25j8Z62M2TjPzxVpiJBYTSR67PGhIXO42QkUqE52RCjKOVnuAi3+h0dm32SS2GEsYBA==", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result_location", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + } + ], + "text": "Vercel AI SDK released a patch update on May 13, 2026.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 256, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_choice": { + "name": "web_search", + "type": "tool" + }, + "tools": [ + { + "max_uses": 1, + "name": "web_search", + "type": "web_search_20250305" + } + ] + }, + "metrics": { + "completion_tokens": 75, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 16216, + "server_tool_use_web_fetch_requests": 0, + "server_tool_use_web_search_requests": 1, + "time_to_first_token": 0, + "tokens": 16291 + } + } + ], + "metadata": { + "operation": "server-tool-use", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-thinking-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with the number only.", + "role": "user" + } + ], + "output": { + "content": [ + { + "signature": "", + "thinking": "The user is asking for 2+2 and wants only the number as a reply.\n\n2+2 = 4\n\nI should reply with just \"4\".", + "type": "thinking" + }, + { + "text": "4", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 2048, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "metrics": { + "completion_tokens": 48, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 49, + "time_to_first_token": 0, + "tokens": 97 + } + } + ], + "metadata": { + "operation": "stream-thinking", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "BETA", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 5, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 + } + } + ], + "metadata": { + "operation": "beta-create", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "beta-stream", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-tool-runner-operation", + "children": [ + { + "name": "anthropic.beta.messages.toolRunner", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 607, + "time_to_first_token": 0, + "tokens": 663 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [ + { + "name": "get_weather.lookup", + "children": [] + } + ], + "input": { + "location": "Paris, France" + }, + "output": "The weather in Paris, France is 18C and sunny.", + "metadata": { + "gen_ai.tool.name": "get_weather", + "provider": "anthropic" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01T5kK4zJvHrkA7kzPucq17G", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } + } + ], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + }, + "metrics": { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + } + ], + "metadata": { + "operation": "beta-tool-runner", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt new file mode 100644 index 000000000..0ae90c13e --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-tree.txt @@ -0,0 +1,775 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds in this dramatic seascape.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 27, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1416 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + ├── anthropic-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 589, + │ "time_to_first_token": 0, + │ "tokens": 645 + │ } + ├── anthropic-server-tool-use-operation + │ metadata: { + │ "operation": "server-tool-use", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "query": "AI SDK release 2026" + │ }, + │ "name": "web_search", + │ "type": "server_tool_use" + │ }, + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "content": [ + │ { + │ "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDH9U/gq18g4Na1FlVBoMlJT5xR7eAcgKy59WIjC9nLo/ymxeFBFi+8LMQFEvGnyikQwA+yxiHHzAK8G3FvYKn9Vqi2KepwsKxDehPsIqzghClHq0zuxC+4QLFBYj8MpV2WNlY/7S5au8l1CCCLZXfQ64nQy0K03pbkNpUh1Cn+czm6XYkXnTvcxx60DHzyKfAgeh/gkerzf+Gndx4xH6UU8TseiVzoC0ynR2UJg0T4P3F4xgp+6tyYSwJHgT6OBHKeSKi1hNTaku0YjdzdHrfxG2sPk9V5vWkJTxS1qMiGi3GRlIsYLfS/K8zU1bMP19XQTzf6Qo4+ZzQq+4medpU1uE5Qjsn7SUK6QaIts8Ruum9b8a+GDpBYihxX/6WWPx+Vrzzq1d78NbUXAVjUT8eflGoWG5HEbl2tTIdz1M8Za6akzdMI0wlsO8Fuz4o/wgt/PUfOkXqxIngdExRmUuZUOCdw5ng9Wn3N1DhlCQQYNCRrmMDXu2ehOoAdoRRaoHbJc6OZlNAX16M/MpfVuh9cLX+hYTi0WJuU/E3Q0cdQW0xJcJWG5oyipVhdeuf7CWa59LRK/DAS6mfZvnqCF27QuWZsIxc5a8PrajGTXtD7JPG0lpgpVJs8EWx9L8h+lsPwNYgNsEErQIIMxA5GDrchHd/PBstxFU9nDZZOlMUe66yIu6V2XCxXWYETttEpQ2UOgtYMelP69PcvHUUfNaHKVeOcv69Zg/La5xPV/uVdniiHNK55sKDiKkYiPd3dnZ4pjSdRq241WHeGPLaEczYLP2hdVeNzCU2jUGKQtYkrAhHfmxjL+k8pO0yNP0U0vJ4svn8XL8f4OlGTXxSTeCPrRiPBs1mC9LaNCqSDBkXhs882EVrkBheOC/B/BUFocdiQptBagJ2YQ6zkjXkElIHxG7dRNG0cr0oT/P9ip1nm+vKN+2Ql5mrDcUETruFsUMu4WbVWWI7Fn/jpqa76K5MtNYe8k0B8OGYqZADi3ei8TjGdzp4AS9/i6gpsJ63GTPYLSu/teZQHkOkzct4/13NF0CwMaqmlsV94YcmKl4hfd/2jGNB8jgWImeFzAcebKJ73B4dX2bGdUiFab5fiQ4Y2WJ2aG9WQs2YMzVqJobL2kZj8dbDTRKzMm4AUuoAQ+RypG6EHGaIlY3PQK7X3Z8OOYOzblqS0rl8hpe8hoa4US2TBPkpzj85nROCZ79KZ4lef04xW+k/NNRqiyOVEszRExRePhAYcTnZj28N/5MxPp/uGmhZW+VzDlIp9cu40GoF/TdhwYMShJKkB6Uuj1wZOEKxmAKPETcRTAXDDWiCMJOxuXJjBzHKWB+v/OUepwjy8PQ9CjkXI3ITpxg8s3lwhGc3ZBWOwWpUVTd+/Mv8YNShnIIXwSrcVwIABPlvklGYvNgkQsf/IuJBoLk/oI6Sy4BKdvQsBODgJjQfnGyuiT+jAoTddR7rDKpDJk+utO4CNdLV19t47sBO7YB2p2sjuZp37onJSRnWwH7zDVaAuA669xEb1u0GXMn7JrFwkpF5gf2Dwlrs1Qt/l3tLTmpRD6RJ0OelL1wTVKU6lYhGAM=", + │ "page_age": null, + │ "title": "Releases · vercel/ai", + │ "type": "web_search_result", + │ "url": "https://github.com/vercel/ai/releases" + │ }, + │ { + │ "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDF3DY5jjLb4AHICGghoMxlCPx/T4A6h1kyaQIjCo+niLHnZ25L3/f4ccF50NkLy0w3wJuEGBeHmuVBt9gQzC4POCkZ59Y/TY4c+3/vcqkRk9xvu8AHT01gVWJqNiUyzlYE4w5iTPach9rwjjP+n43wfJBiyfky+jXzepIV2ADLYYt4F7rxBfYkHAaQuXy0dXUOZlq4g5Dr/ojVJafscugAabDKf1gTg9jYrZQeE8x67MdJASrrAov3QPraN9sejiMmTrENHTQzkkOzOu4/uMz7b05Vpm3YlVn2735GunbupA42t/CLtzdeNZxOq51CuUBct2ELj+sZmJiQatS91DFPGmXnBQOSBfIYMClhVTcYbZiSoBaa8X0MCsvmz3nX4kzCRBGAvGdH/3ckhjz/1g4R6/2DOPPHxOI8gaI13wCnVCAL9WgbqY+0wq+vd2orisYemfNCrlZZ/PI1YyPn3+vvKwMkI0CsvoWxyOtehurxcGu6mng4Pqk+Y2QKoryfbKvcsFBIWktxEzEzHrdh68Pk06QbDPE3Vtx8wEBxsLfmSJx2k544E5yi4IabVN5nR0byucbxQV94/kJu5XrvCmWFOR77d0PK8KrIYEYF4r516sVY1xd1n9EHph3yirtuMpbBMU5K/xXqSHDWLhGZHNe6DQJ3B8EbUmRBgD4XslpVwXaz+oAZbmA4TvbMV/ShtMCEpOi96cTp5umEaHPWRtucs3aFRWTKZAYUHdvM3p6cCvw6M8DpvWawWaOiPawLxf1nGgjadznVoAFPJmXDchheDMf62hPh634/YP02vRcKOylhloVvT4lifCuepZhXZRqsyFhYZCKEB73XlrV0KiK8TRaoiZsdv/texWfkO7bbXP+vcSRCQBl8U0PKUk9vDJt/me3BZp72V2o5TdizXP23FhF54HLFgWe5nH9ktzNzaG6me/544kxDWsXOJOKoGL68FiEnqrYY9HfgKAflv7FpCZbDqIHZAzBncSraUYlja7w7lIRgC9fc0gkG0DsMujJtGfhNvav6Qvpryi9hqax5W2hMj5HJTa9vC4G5LZwWisC04NG2590cFgh8CAlVxF9Rz6jLqm9SV9RkuwVCkFyUwQjWjlBsBOHlkHnSHYeIsP+gBot9iIMUxbob2iyv9txStjBoJCzbkXxrTSBK8WiRcmHEVrklwcVB2fc/YzTtCm/uTijjRIWncggm6+STokJ8zDb+E+21UD0ntKoMhbUOp9WJGV9kSrNcBHB/lxNcuxFZijRc3oJEKF5lObwPGqMife9DoFCoyBs2Yuhmwj2qsTNExXXdBisWXiU6Jg77+uEOZlKF3BcBz1gTTpWRRvYISk5/em8zV0QPIYLXZ3JuIbSOenWWKOOeQvwi8wipV8EJ4JnpCM/1tKBhcmJ8oW+d8UiEV9GH9i/1maiEZkbiy4CEtlSvQHhRScE3N+fD5ADGCg35hvIQQ8rrKMD6okihI+UOtd8Z7r6bS9sIm/XqX8Cb1Y3CDYlSjYkxO6hZAMny6F8HwUz4dhIJdf018FCs9rfA9AiS+HcZaAn+PL/mjN4l9tVYR8/xPolMHbMJtBAz10s5ppuBUTM4atpgOdNCjbYgGBAOsr3HePL9265DMbtMLviHcM21elo4qMF4m0aZ4oyMlgdedlgs4h30nX1WtAWz17+NzFrVjaDHI4AuHRFA4O6zvQPP0R++OzQ/FjQnUh1TbdkG3+iZPaBVNWaXk6ftOWIE/tczU8CSLu0Duxd2JaHMJIUnS8EVHt2Lp7wBUG+yLDJZXc+kRqdJdqchNzettzxGWNwpDKCiL0bD475rRs9dTruyYh4FvGl0/czPDsvlGDe/eTQRqj+syAt8LzeCJzn5pEwwuetn4g5rkTNkgv4zRBotc6d3mOc783zeVPlDcIiE1wvyrKtoYGJ5sGpzEIGrWn/sNOlG3ZBfCsvg4WqqTMmNmcOZQAUCj/dAC1QjFmz6M+LcoqtvGzASQh2nbbp2lJd5yrvtYubUDICuizNWKwSemXRLwHAghIwCnMm2wIvK6dvcnwBTpNGba9v1U+CHIl49zdq9Zgb4c4Fso4a1YKqGXiwqLy8rH9IscdQk1cA/WYgeCcXNJdLYLeq1aWWc78Ljv2gtRmCqKCq+Ykk+TnBr69Z8qcvMUAjrLHKY/xYaUOZr39KaPQDYBhZEn0yLl9r+7XTPbahrotPOYfU08oV1XSgEKnMsMV6kkPzLkPhRPhJ55h4qoVQZs+Ri+05lRvPkm2HmMSlLD2/5e7rPMYDj1XdPJWQ6+M36F/paezQDfMNxmf0VbAzKGDcmJqnuPLmuhcCY5MiY5+9ZHsJXq+9WTKnRBChnTvr9nuYkFMP76p9zsEWH7mcVY3tjuo3mBmjSxEBX1lZpesCTvUREKkLjsq5ICK+Yf4cJ1TUlrsUaqiLPDa8oZY1LFA1N/R/ylqd3xCbh9NO7JZwfQCPpRySvXXqe5t0yrfkQjh5ylF58FwP6O1UACb4C8M4JKQhRMR0udeB15I8tCmTa9S07TiuZas5RUt6yF93ywQdsFjIG70KGvrk+NUA8LmbtD5n2XMXiUfqWFKDUhVQPYd1XOqFOZmw1vs7K8T8uaxTeGuWSuhtsvA0en3Uey86JQ3FXI4JfpbRGK5wxXwbHX6lxgkxM7TS8r9T5Uxcn54HNaIB5hA1DtmkQAP8K43PPdwinMILx32dSe1KuQNDguEHeKc6Hj9S7LfJwgNG3QKjmxLgQblBTs/SKMZHGr2fU8JHppeaNSgWSZ0hfKTdrIzmeJ/+ualHTA+eJXdt+woqhIP9K1rvvCy399aRaTPXoHFvGw3f4p/CunDsFt7xeuIbeP7atvf5/UnLeXRG0hVD3GdeksfFpI0fCRkSfW9EhHRQAkTGGRBzilXRqmTB0dtMmqCGGyDXPRu9DkUgxeau1/ZDDbjEutrKG0GJyX9YHLbmDFKwSTnoeiphD4getBs7aOjs9cfjg8GDxQTWQ6AULGJva5z4O9yfM9jV+drze4PSPZJeBEtfyN5CUPGEWFD1Pvfxs2uVEZ5zkDR6KAofXIzW13SemKDJG3f/wYRtrl7ynOd9sSX1XwdXn7M3yMf/+QgWLar2MXJ7D5ga+9QlTrrxF+a0mWED8ZfDJ1P0MLk0pHplTFYfZth/XARrbUbVgCmX7lmkXeN8RCvm34jUuVgGAeKeqWhj4/1j9xPP1Hr0Zq1RZXtVoPusgFS7UmJwnOzs0wwI4a5Psep6nFsLn+dJFpgSAgJ4P0bYsOxXnSpbZkXbNlmVYl0Ntf5GVkDMbf0vIbLQZtOIe3ClraK11AGh9M7UlN1DDQ0E4o7I3xKrb5QCjYAD428XdKOm25sIBuZUZlcUqKF7diU3qiRs4OPXTWG2MYC8G+GDL7BjE1Tzr3MxZ2WFQ7jHa9X0MvZL0F+7spnX2aht0oJ+wyXakHyxgqxqn217FYpFrcaNCVew9EtU2w4nZPBx7dUFX3nhSWc5knr9D7oh1EQmwfWID3pykAQz3jFG1UXUeaUPkvYFTMCAIqMSlNMuxEgXC0kCM7aSzKNW44GOWnWl7GhZlppO7CTJLx3ac2mhhwP5q5a0qu62rATz3V9SCpXQPKmYpq0D0a7hLLz6tsrnm0I/gkCK5xTlwKsQF9n+w0Ha5nRcNFBJUEtgropRV+A4N3dskVeTgbzRUoO4LAkx1Ma04o5uek+sFAX1rYLQOYXFkY3W9k/ZeDsr4eGolrjC131jlRo8WcVt6stR3dgIY3de3armMNFKkm6ZQJgh7yBPhJivs7vwpqw5SCDt7tF0DfxJPY3rv0XZU042kpbQg5U7kjngI0oK2uB/NZath94q16VGIK2yME7bBylsPhqg4MlHYzYzTjoWJpLBjAAQoFdAOl+QbHf7WXgJ4FAyaoerw1uGfMQkmA3DwjEFS4x9qsSO4+JfHXItw/3mvt/gHgwhcS9HP8LXCZZeOzStiLvoD7otnHCBbECAa0ekKfqWcHRgZgiJh8SHHrZTMHihkqdcJbSE9n+XTpgOa8Cd8yAaC4FsjEFxQp/wfzBdC4GwPrpCZVriaWp/lYAEbihZuM2HYE7zD2G9p1Sfdx3X51B4JiplWDHA143wtFoscbWuhhkgK8DFAsrWR4Q2MozJFevnn7Tqav3DB1FkgV5gkgwFVt/ob+6hICOzMVNvisCwTuoFtMbgz2S2H8n+7xdLiozC4xZuBPSfndqlzJfHRRQpxEKQc+ukzPrr+v206bKchgcT9xCNYY71rn4QTpXD/G8scWdk2YdGc33wSBIIFKe7zeGY1LBbDe7V4aMskKZqOMdxsSflqJzaIsFJFWJj6rd3Ij6LVLl+URaKWAUjYhSrdxCLXIfJRMC79xDxH9A/2lzyCZE5RpPS555tjVWGAM=", + │ "page_age": "2 days ago", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ }, + │ { + │ "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDMBv9686WpN4/798eRoMkdYeQ6usG9C4FJBOIjCIc+6zBd0v4jLwOCFLej+uCHSZCavqu6rxeKA5RGFoioiBDg2IGtTfDB9tAvI3MlIq0x4b5+CGRb/YgxUpDj2lSSDlSJ9PLYMDy9urLLfoqFSuCFO55wWoIo8QuDQcKfIZ8OFB6utRapLUYkq/gGOM/c7Gr9kup2Ot7scC9LE2RkfyIEUyroq0u5mVXp4SRZhaaUFvW5fxqzH+C+0uaTuswePRO0urSNCO1yYdK/3y/+yDItaqyPsYTmSz4T5aeLHBJ2ZRMdHX2m2msuc1AAWxDX91sGAeTfGihfUVmgnx1P4zU4yB3C7710IZBv4ZbhUTgZWNq4MbQWv7ZeVSK1y6iYi7p/3XYplttfpp+C/nZhgkUij6B1VI4WtsIZCpJKIlI35eAp3mwxzv9fHRoAipijcv0481dmWm093Tri1W+ILWv+f4FqI2RDfdOo3ci4cyflnEqZJFewT3364sbu04k2S8MS7VXDEyG0+o77A5ZeNdXEEle2baRea324PnkqXdDfcWFjW0b+CNwjSvDKkWYFUuC/5dEcnIirjE4WO67Gm9vbc8QjRGiLOslKDDxyolHItrwXKjm8VSZQt+nN22EEjsWNKlF7u66aGYONqjUEVPK3a8pagFV8sKy/A4y7mDDgmoUB3D0JMrmDPgYY8AaEmzWNkEMfIAYpihXyCOCcPw5EqiQc9Ja7tTIEw74LpeAYZjZJQi+/t21KrYQmYEqriaUUY5+bJ5J1/VKuP+tqT8/1SIE/UeAfx++baR7VzruMzPAXHa9cX5h2vm6HbWzz6Tk2oAx24vXSP82OI3eh5owKZe8ZAtqeRQLxElHj9bNq1iek3GcLy6a43Yko5uZApyENICDqQaSw/eVQP4XcQQR7Bf94bj4r3Z/RvQvyGEDwH+Buak0OetPH2ZDUjp6yRo7xbguW2MohP/xI0JyMNDMfMbepG2QD64/iVX7GjC3PCGAzvJxVCiyEb/OqxMHnm5HDQLWGLYDUDYlrXdgZnatcd1//RLSGT4sCRraK+3p7ZmAHNkylBDVxyccg+TFvs5REpB0X6P0h2wfIP8azf7Ctq2FLdSBeftuneKbNHvhaqKmz0Mx4GXWY8inHtwu4fyc2y05spDVWfIToNc9rAxOQZFJn2NaprfQf9QTG5qlkPnl4VYHf/v/rDjyJYxJeLM0a0qLkqDQRQtOdOs+yUiVsjpJvR+uZRhA5DDArk+fhWMjOFLuDIqztXozf25x0wCGzDey5+NYdkqL12X5cg2/AHUXCZGYTOcfWJrG51sz6K+TuKqiXdTpU6rKdjowWNwAVK0rLxfEOwIHH7qoqdnv9i25I4rEjA0cTJavNgjvifuhFGpZlakQ50UliwjUithKM1BErP7h4UMkhYCso2/XokGPoMrV271pe229V9epm0lF9NUsphjVWUFp2tfWTFEOXmsp+qHSDn0B/4/aES4HFYYmn3j7/eDe0m2dfsLnxf2oubxPUum4OWUqpDOPx80YMQUeE2nPDZC0ZNsi4PgwD5wUcRxY56Csuc214P3zKRfXJPb/Jf8eNdKNpshPl7Z0IjSpyHKApVG+eb8gKQre4cRzBDPFRmYQLk/y/570CnBiYI+Q9i8XQYG5dTwBkYKX9NaGC2nY9XdnxX8z79NyAw2R89i7pikixAOBfK4BQB2rLwJqFaYLFynyb4wp9tss60B0nTJUQfIby+zJH7bh79tdnjmR1BM8frcrr+28rA7QV6Q+Jv4exQabpCMrWmdZSYFwqvil79YzUG9A+maHRtznw9BKDXVv3+pejWp8VwR9991NM/wPDinSKzcoRnfzV0E82JSYtE/F+qcNvnef/LX8XkuokEd00ljV46RK4gwZRIeGDGrTMXg27fs7yvPPdbm7ijW+YpxrqIAv1Z3HNXjOxCPmPTpzLf/+YFMZ2S99CX9bNpRtoKknyululW+McERJxJWKwKUVY/kJOkelftch/Df57/b9sjXqEVTm17kLCm1zdmQj8h+P5CFA3lind6JJ8jIiSa6WI3/p2wq4CnoFKiE5KrhwWuso7BJ4qMF26oGZ4CwRTaVEGgWxe7QCiWRvfCPJPinS5FQfWydYh4RkxqwlqgLh/EFuRdNKbQa03PtuMXXrhkihVbYHGus1oc9QBGe/FqjArpQNIudE7SA80gOcLxSd2qd8Ecjwk5+3IbyK/JcnC3AEa5MhSTboAjpgj4PG1rnZnU4Ua4kETCr7QLy2KgnIS8LhE1Fy+D4OwUdA03gp/xIwseLVZ/p2aHeiSt0mGRTkqhPXUVE+eGp5uDEsr4eXo/2h7gljilQkCz2rt3JD6SiaZe3PKYk+dMHzAZNT4AyDg4tMR3Ec31r0ZjIRZSPdk4brX8pyENNQfHUg+cyW9fV8qVMZneW/Tj+w2kS5telkBDOloQs9nxbmtnvyZPncOtdPf06xdplhDSa2NbbtTCmzkituNKInZVw/wCtGGLCoiewgHl8gMhZ3at9w1V9T2kQ9KUIMMJBf+qCI8VLLoFqWQcssCb/kovAepQT/jWCDAqELm1Y7dcdRikdt5ZlcOJJW5/ZY6+cFDoww8neMZqNAj+9a20vKii/a6+SDZDnDPNYplZn9bhPG0zIUIp5Jb4n3QhGV2kAxr3wDHhG1YiMyO3mpgh7zQ8aPYWXKsqUNj68lZoW+3q687Fma1TDH/BAth0mGPihc9LSJkOfcMO8yYqHJFv+hFgriGjFg3bqM1FcSgnUVYAKe+39+jVF1KorSJ0XGUzDg3RIGAOq/FTvvKNCfQ8ssTQVv1uVh5pVsdwdNBaHz9cUnHqoEVXsiXMaZjkmbT7vIV1ogNDe0pQO4t9ikvRfkse5ICDm9FW39a/9qJuNfduYbohDS6deC45R2UdQrA/HViVZFp9Dox76TzGvarRll5w0MhZ/MNbxUhu80vLb+pkQYT8GGvEjf7lSzRfw5x9WQmktWIN4aOeoy10QHJaaabEcV8M1Ymq2IPBFC8zMYV1upD4rRsNBx4jL/1sd71e3TW60ga8r8L2+0bqyJpXsbvtyrVF+jDUzEvcxvEAZUDMRLRPZcXf1xFHDRr3zRZ123P1+ceeMgLShKRphgO7mrV3ij5INrbJYjaURpl4jM4/EAvEBrXUKXCv7TSLOG/Kxpsqr824tU4tvjbiZDxfsqLb/n6Ixraow96m4letDFWtqdrkuGh2Nspbo7gVjiiPQolvMspQcbpSnFw9wiiMe9OQc9l/hyvA1otihvIphHoyLkiiGMJd0jYqrQWwo1TV2mW9vrq+S5k82DBKtRWtVoc0MMAA6XKtwnFUtIlI7rHpdj9xflO066dHTKHzq9Z05Y9hfCHPWIPgqwHqELwa+bvsoWKEbZpEAxZsQzCgumx2P+hiU3Sn2omdP16Bzul9sDsVn6OlSWg+fAkGPpPFNlkkoVa1ozPKkybzqLDCqmU3gVd4KPhjQbqh8RcoHpOK8miH31Y3Hy2y/Q/RyvEXAhRFePXn7VR/QSNlE+1YEcuhsAnnkZtXgtMCrJSY2/7/OUzzF5+GI7wuQrJjF/WTs0aYTNp2pnN31oKmwDNWneI1owLMMdxDTdyq322c6IkMElsenXk+tHd8a5k8b7Vpld+9d0FoIH/nIF8Q51dJmw6qRYq4WTukJPGnEBNbTrs0Tnc0ZjdzKbmQUT0cfpP9+xexUmfGX6pVuBT6dMbFKQVblCtIbaegBK0KaDWtlElHJ7ktwSINRr/kM0mTd/CSzxcwR6nzdLBfnoegwn1vOOb4clNC4YRuKF6+gBAGvsAi+bLNOu7x3s1DqcIHoajn5SHXQukCY63+RSkiY2rMMZJrpn8+6sbV0iX8NXabQk2Yn6iSG8jegCd/phV31IobDBXD7BfkKIq+t3kJjzJ5QTeK0HYeVFUwB6FvJcm61ig1dBEOrog/Q3mQ35hnVlnAg1FsxUMsoawbsC2goo+zn/kAdT1AaxihYRjZ68VmBLTrMFgCNxBb3CYgBO49nhUa+hsgy1++oYGo0gvFrNiLgWJjG1I9FImHnMVhRCx+qyAkNu3j2tt4FkKo7ETBJHeblkXS5SlaU6QqCOt/d3Ma4JVeOyZEeMr0zmNlQjHMXSo/2nFlwb4D+IgYu+KCEx/86jsS7pSA8lC/jDd02NiBI5hGWOcZMNi8KqSzd9iSCTUfpTs3VXZsppOEZOJeBsbEUMSwPw4xpB4EtGHeHL+oSO/pFdPDLokm+tzjxrTnlrkBCRfXgNWzM33AwzPTegGIeQnIga2zSa8qCuibaorUvjueeF0afX802A8aHx64LLTaZxrpHrX85giseF9nxUHeiDm0b3FphNzSqDseRUGUvJjQXN8FngFI0yhfayfEZDuxKIUThiNzaP3swHfHMTi/zTX6RaJ/kXm1mdh48bTY6IZo8PwsFwgReEmviX2EETQgBJSarzM+G/J479JZrukbSgwk+Pb6qUUA4zPGfChLC7faQxkojIr0shjZUXqHpspDwSn2rtEXTAXGOcm/pN1CJNZ1DpXoeoxNZt6w1dghZGLQ0FwUeqPp6kcxRkPewT4y23dWpOxiM5HcKYbiotrr03kvQo8IWkufmQ3tpgtN24qdVft3nqB1GG/s4EWkPwXFM38KspgHvK2bOL6RtZvMj0G84rQhXZ+b618uRuxUYRvPixdICleasobCGStJHgG7sPa1SMnb6eW5zNXnMZtkL7ibEpq4Qy+g+p9AOxkNMt1o2dsa4wHltm+omytzYskZUpaydaI3KZ9b4vVJp+ND9g95SjN7Fr3ui6l9M6spHEespvSXfBGSfR+JAS6SYp9lZ6mLratrRI3iTJ+LlJ2QM+JAbjngub0QVm77SyIiLicDNIZ/OZL1u0vAAEUTaWuvfwk1LraxmiJDwOjELqw0al0N/hL5lcP+6+K/qIEvSveagn3h5MdlpI6oMlWdP0KEQqVwHLqWvwrTL+INAzw3rABEmIkmqMXtzuT/b6d9PBtqJhrgFZoamO1XNp0bcKKqBUKCDmrfsOdtHiGauE1axbf9pisFhZYH3N++NRb4FGV3yXSPLKANU49g4nc/eBXxSICPYfi+dveigI902xRr0pxF5TJ+fyVWuG1yntCc9bsf0cGJCzRE9Jg9TVoFyWm3jpjUHAjWSoxdjeH4rcasq+G5AXxfUBUa0KfuSV+tEwxeLPhp2SFD27g7l9zBBu6hK31VTM1BdFRgJCuVWzB+MpGMMMdGqKxBDyMWnd1wUHqUBPnle0Grs1bUlLPTR+gEGGfY1iEOAe4L8Y1l8bnCzjf1M4Q7unurdQWm1/NvHwMCLIxgD", + │ "page_age": null, + │ "title": "Release Notes | SAP Cloud SDK for AI", + │ "type": "web_search_result", + │ "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + │ }, + │ { + │ "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPLrkGNo81n+yBpyyBoMFUod2jgVEruDGYN3IjDWVDwJtkvLNYBHmagFxyBmjB5ndX3y+MCNQlxBRijGW7ECTYr321pqMGmtqV/LEbUqyhSljjzzEkkPQGdOBq5lBMGUNueTufy+Li2bk7McQSm3C2lpazpWb8bAXShoTYRF4bJ4g3uhnFmv/N+fuyEhGVTvkD+xeyX9p9x1GZ/7+j8asP76yJ7/ns3Raq9eQacoXEdkzU3IETTBhH420hwajIEb5+FB3lOZ3IiAyJtBqjQPk2HcYV6YNQtx5QWvWLiyLNcADfJfxdeECjTvZc0+Lus2FpC+U7e+0AcMyg5KSIt/L1aS19+1mOHa1gBq0rWUp8PFu5XBo95hqE/z3xHYC9BTwscOEVHfvnfaPpBUCtmaR0QhzSFBDbUrPMIZXvQuEx07rNxIWccdpuuj00XJG8R8wRi3+0LvKYO65Xc3zFQWV7iYp6yLGe79zpoR19bhy/1CCNH/CAERmJBQDpagmenuTN8iaElvbN7UCtvIyO9yMuNS7OtMAmgCYB1cgmWd1qyL+ky1/cxEwllAyUi9rhK7kBNgf4KR6M2l53vDpjumoXGi4MXNM7CxqMjeu0jflRs2qZzzriMdHYtJRrKUwKdr9AUWFRii8hnizWLity3sm3UaVShtrkDM+jnZLoLBEle/Ql+sm2r+Y5JJM4ZcQzksvofIcX8ONEjMh0jVMotCO5/2UW4+VOO44DNc2yHZZUilKCmY0x01rs+Oi2IxPQfvgG2y3uX/SoDbb+tD9JxIVG2j8JmET+gfonD0IipNeDJ/venrZTDYZUdnY381nPnA0Bfzyj1LjeyhXdrlwb9U/Nfn5WZkpo2nK0UDYRM1xb8bQwRE82CuvFpDcS5Jvby/9X49ZmG7K+ic3dy+XRYD/vcL5HnMFxXGRuUOgchtVVNi406v2InEbhC7X2/MvhnhBfsCrNNonwKcN7bz6lGZhHLDNpG7Ngh+ASjfDBkufvDQ71UaD0oENs/TTv5JH1u+H8nFi4AnlnbCP/NqQlTmOr7BOyCnL2FgUcHWo88DujaOtAVUyFmyU3xb6yZeX/xpeRD0P2xOINWTMV39OD/fCvTaiRcBW4e+0UjwoCJyXbjf62eAC/dkO33bby4WQBTzymk/HCjgmlbtN7OaAl7+BYLkLEomz5byG+P1mHaz9T2qzjPEsBHD7HEC/jK6aI4rQ42ir3zAXOs7+s71dzZVIV+Pnvqo2yl5m1EckLRJmDuKdVcYrucvsKMqkeTVIvlzZK3ctNeI0S5lNvXTYLw0RUl5c3z0dDieOaGgWLO9DBN3xDWzLixA2R9jDUAHO9NqeVaahT6m3FLFpeZK8F397u367hcEXfSLmZSuIe7vdIcBzaLIDkbTD2qQRZ/+39bDIxM6yYgqbhz3gt9X5eyE60eXH6j6zrK4YnPfIbz0JUZ+3r80eqdVsjkSbdqDdUPt9INoHMpmzx4Rz8itTQGfONi1IdRgjb4kIZS3HNAjGqlNeGSVMO0tsWhqUpDWkgrxwg78fyctAGEQZr5/W9JJgkxUg9zQXkIppya3Floso2YNFGhS4fHFsnAYL93bfsgZ5ql9QAqpRQHCSK40BKAqAku7/L8PJhbrK8UcYr0SGvfv4e872nSqyY+ohDUcKT5M2TXcPAjPveTZqJHqO5AWI6ZcTxa62lZxM5YNRoVhpvQfELsO8eL65SoPi9MZiyUaj5ay/byl0l98EXjWxXxVgEfiWfDtlw3gp3gfE2zulFNmsIG6asfg+Jgp6a8TwZ2Q3MCgomroV/EYJueveWZQl1YpF+HUo5JpVQ0ZX/bS3oyEBOCgZC55kY0mh28YwGnqkDXqAXSHGwdC4NlYOCUnvFysAWlmOrCeUrzsgnWAOmlRK1r5RmOoWsr5nrx5igwWYsyddlFdsbxAhR+wWC2yl28kC300RN1R6gd260kr6fuKAa4mqoxB/eLKOvgom9D+YsTiORAWyZAN3Mupf1thV5t9mX0XxfyKndBMUjfzkklz64oidLF1/DBReFv8UVOjwbe7x+beZdxExhzKRmKqudr2lbEYjmVxYm7kwTKmKy6VVsF0+g5soa7lc3mGkllFRbvCY6RmBblcj/z1mZtK0Y2vwMfPtW/EkuX1tTLSuoXmF23jmZEnUVbOOkoqe7sPU4eWwRICuyjsUmQj6elbU35zQ69zMG3IFBVUZtOxjp+TL35QbD9M/p2/2vDOEfMOM5hBg7q0Di8rmxv+wUs/GFy1cILnWSvObau2Qp2TQcCzhzT6lo7fXxbRiqKC95RUZOfvD2fUdL496LTboIXIsQsPlzxufOxp01A1/e6L/Tf+8B7s5hDbMqC0z6GQdCreeNHG4c6pJrzk72bvyxynpel4t5y7qLJAFqih5dLqW//F5Y10QWfU/iY1hyx8K9HyNaLqcTbQg/DH9eNeWRhqCPGGKtzAhkOa4tD/0BpK8Uw5c7aLIJlVsNeSDWcDYLGvNVyhcWhC3T9xs6qpOM/BtREy3reohNhnp/V8o8/X5gd0LDJrG6IdG1Sv4dpPjhQbdH6UOIXNWF/dJ6h/L7XGgu8Gm1GQ3JiIKmHwRjLG8XPS2Gtl8hQvJrSqJSWwxw0tMMAXKdz+LnUHOfM78x09L1d70g562WD4J4pYXf7qTDzA9FL5UrJRZN4u9/Fgz6J5P2PrEKsQanXAktpoHtTDTF6m1/fLtQErzgjtVDR2QrDY65QpLYS0yw80mKeMrUlDgCUGlMXkCepuuFd6Vy12SKVOlAGg7yD0Td4aNu8+x3ksXZLzfp0QcDW+2MnkGgZahDANBdh2ONCbMSRGJDzU4YJa1ZXghuffRwdJpujAlRlaN8G7U1yBDmcI72dCVOLNnG3txUyTj448sB/RixMLCfXvqFj1pyIBCedTKJMjiAi+W+2fTU4jjJeTK6OAB817A1vaLBVWjHzuAaf8i5n3oDocL3og+SAo9QkWm26PVEsmlpUzlrh0O3+R5JELxt8WclF+kBqFoV+xmoruhdCDKXq58aIzstxut4SsuWSMG9I/ScNDh4YElY0PUWFBX5F6lFQKN1MLIoHPCajODAFQcigBZUvUonqo9eJjzKzl6KQaI+aF5hJrDtzJeIXb69apwcTrCb9EEpQdZ0OdAcg1frfOnkRpO9viZvQcbhjK53iTSi6oJDnhg6ZbFp5bQTW15FLQEXbLexEInZqiftsCb/StBXMdevz6wixvNA3t3FZynp8/NMzJdnzsL9+gqse3ZCTJMEuNfP2cTzONduzPs3EqgjBxxDJaryjm7A/MrP4VsjCuqJDuA7pN1ASRIjXlI+HuCNu7MnPss/iiuco8Z8Vdp2fqrLz4+rja19dWen/okCXl/C2jW5aTv5rGszo0R/w4RYgl2Vo+4IlpChzYc37ZWX1oFKKVrPNOyPIW7T5so9NgordUqL6f7efP/hfFMSPnsPxVKpWw05FMiDk0r772PFRO1FPCZmRnYhM0bJUXClwx+EW7j2560161yZwPGjeuLIkj/48R9xL6N2kzR1bLPMLWs/LepYv4tDsf/2t+9xZNRwMMPF8wVPFKOgRcExUj+yrEt9oEpAJXLf2ryNT2Hl4z7MFn1+oYAw==", + │ "page_age": "1 month ago", + │ "title": "The next evolution of the Agents SDK | OpenAI", + │ "type": "web_search_result", + │ "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + │ }, + │ { + │ "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDI7mMDzu6EuE1sfZQRoMggo9OxaALayAnV3YIjALxrR8LDmldK71rzzZuCGQ/5DUxJ0pd+CgXiQTQn5wJKnSh6B+wKVsxF7zRM+RdzYq6x0koKeTQ8sTLL8HELygBAjAF5aItnE+6X2V8D1neYAczEW+g4zx3iYpyHng3rQCBkdbE409z4byO2f+W2+WBSUK/k1Rv5Q5UNxLzYqTqUXydT6TnB/lHADY0dPBdG5Nj/shYvuszWLq+fbEz73XuNkpUctQc0SiQRexiztplojR+LG/Nd3Ek8Fhnqa0zuZ7j+ZqpdM3zfcJYXQI6P0V1HqRkowcHRhROURswiPq+k4dqi70K60QOP+DQV8WON37q2rwOvOR3iMnpk6sNZUBVAkbk7cPvQUSzP6xjMzDvubpOyenlPOK8OZuIDyhVF0M/f7cnF/b2oDM0mDfkLFBYI1SMkTe7ezrG5uHDDI0nNQnN2fIGq6/VRCLFoA9EPat60K5ia8YFZX1++SPk+cyk/9SDuJxOvgl1xaQRGtXuLpD6RAJO6O1weJbSUD9sONfQznILkd9FWJ0WshVeuTTvKkFxWkThbsBfSTDO8OjWhr8JVAGsC7Jhq03uX4S9jym7bp2Paxs5/aIKUzSRPYkrJ+ObQ3K18xvYeZiPwheQBswrWTfPo8e8ZWabzZj5kRKt5qkPRdD+//792VagiY5e/cEb997DU5zmHcICw4enfAyj69dxhSYRCq0zuvf8a95UrYrv1zH8FjdUeoluaZtsrkOZDuykl5MIaxq89X9bIyjDBWVerFBFXIJixsQUyV1ZrRsUNj+mGLJB2a1K7L69njQ+1n11yB9PWS4IvJfUFZt/ebRoQOb6VXMNykzUkEzQVaQ9MFiCt2QHbCPqsl+tducv/TIfm1SlVao/9M9K0+nqI6V6GqXgxW2yzEbrpBl3UQG1QPe0ThJmwprfdfU9pItNSNSfiSaiyFYR7XG3INR7+7FHhE1+x2h4NRCn+6uA4nogaCqeM/ZID5SKp0BbmRiWJHNwP1f8e/NSs/i9oBZWhlUXjIxUnlW9zAeBFdBF/7GWixbXWQXzHyqa0237oTdQqFrBKsEE4fvky/w3lmOBDl/SNPFIoUvcx5dMxw9o0cZTekBaJ5EqdOBAMhCIrHd4/c34G7pf0Urgi0OxbQLarXzU9QpyR+GeLZmoEeGWIaeWMZMY2KP0pDI9mbtAwSCkR8j/l2nXanN7VxVfF6Ysqih6yvvArKhqKh4+lCAaKy2u9bgVgrWT8ufQiUrqtZPze3IZ5Uc26fMJHcRNvQ1uTreD+LArMMuMUtzQjIsmhdvSHftqrOrLTvbHO2cQDGy2tULZv11ROliRRj2KPtOOxSpG0j8kJtmOiqFRv36XPj3wk34AaOdDYosxGez3k+RcxGYgUYakEdi+YybnUnDLbBH3ltoGGkOhiLQF2PuKeEw72IHsicrgD3YS/lEymiCJQqJoXtZjBedp6yqgI54cktrMe664FVAYLURHeIyOX2vhnHe6iB1xta10fARoD55mRM2v5fkN2sad7drM26tuqycHvfwsoUe89SoCQBb5IO+LVYGvBqnA+MaFR7hJp1I8z8bDUDX3MnRxb89c/Pt/UPFpN5r0FxU9e52rJIo4l92x6ysqSSsixEa/HAtNSc+JzyFlcLlPmOQY0MsZ0HrdMIgDh4uVaicVuzBazSek38YvL4hY/pZnDb0NlPrWOwE9CcUlFvnCIvZbsrjLfrwZCVYpfFTOdQV6WReInjpBzrg/WgrTjK8O6HwW37gm8mkSCJ15z3E+f7BvoC2kwgz+vpgZNLVUa5AwoNv81IERrj4qRXhLUtJFEw7l8uiYYKM2uXFq9AnGbaM7/neBjAnS5+LMVhvJ9iK6XMHWHk+9rifQ+ttWUF1eZ5RnwdvIr5WrIcCT6Fs4olJkaZB6nrk8vvb9yrkbKooFvl7w6U9MGmdDQ3RMVqI7gw+MPPqWIhKswW5Z5dY8er2nr5BxT+ERtBvENsfqae6YBhrsGlRKGzrb1CG8YHT6/8Q8zam2GpGeMazOI8neDuafbAsQvOhp9LBSr2evnZLdrDhGkCEvuLssSWiVIk7fcT7To9EqXvFu6h3l7L/HqirgRu3Q2cLOSyQvilljQ8EawKNm1jEbQqRSP0ag04xSOLVlnTY73rNXG5pB/hgLvzu166VAAlOkoPXMtGMV2n2n8YBUof9iYud0O5KGY6GrdiUbtqyRL2+xRXjjqa0MoTxSri5fo064wTHSkl22u5qn6OPocAizwv4mKmyyD9e5KtnVC3X7Ws4xjWEYDX0S7yd3AErP58lP3GvCt5mNxHYhawoQHRDv8yH4XYPmKZ+wuZgdtu110MwCsYHU9w4sB18bhAg4NYBXxQmLWLvp3/gZac4Gt3wpBsFhXc+MTstM4kUiJwFc9b/wNY8bBpkFucUgZfcC7D/Gq/Zddzjb5QCB46e5ceQkMw4D4J8MVw366akOwH+ZZl2dypzxhpsFx7mzW4XBTFqhkAxVzAKNbzKpvgOHX909UiNj9WNfcB3pHu7hFliVZhh+8sH0K+0BXxGoKxnwVXQLXQrGXgGa7MPLxTEuLyHV54Jlrtb/9yizhdjDW/u4/TMhVzcKeUJMGGm4+sQxrlwyzZVLjRS8c9l+7u2fF1G/MXaz+PIQizeC8DTZXky/kdhXOsNlRf4+oqPTmNvBuNWDHkih9cvdblNmXBdg2gJgGCOCS1se530aCOUBe/NiDe4QuNywu5MfSuCHCXs24SEKrNCObly5TopWVYCVoGK6Wml75t1Uo47cybwL7MSJ2MHZuxUWh+XWI+Gmb8ZWrwbbkt6wCPXyQddqEW8fzHQwGS/HrxQYE0kEiBa6pE6bK5zcxMQD5D9Um2jIKts/g2g/gN8XpdYAiNKaW/edH9VoemuVCtmksO7vL2DNUzO+yFvW1dbhszclvayrCJf34DscKgqwI63iPc8J4GqA4UeuDE3WdtOXTQolT6/gaxYgQioQwOR1cZsPnDiWoszBUbCLfOFj3fPuxtXoRVbprh5sRhTUrGxHUmMwj0GnE3Gte/6bhtmIWRkSBDF7KuylaXCRuLs0VTNHPqhzwofvBjvP5GomS2QhSnCiDqUoD0gNhnm/j2h2uejaCaXP0gie9pi4xLLW15xyUIKwO4+yP5htC958+5MAvDV30c+K9iMLUMw1gtnQraF09w/RT5l2Qeda1sCI/oCeLOeQhmzAh6FZVBbx7xUJm9kx9t+8Sz/Qp1KGlOhNLBABoKnF2XqLtSPkV+GzCDoLY7Hr4+smN2oKFIBP5RifseM1wmdNI6qfw9b0CnNQvIDlMZC3r/Lze5afaStgF+U6p3D/ZyDu5nHKXz5SsOmLdN3ig4kvlKteX9EUld0w7aCdhWvio+oz4gSwXQRwrCNz5Z5jlDPXaOwoxIe5NIrBwSZ7RYwOprCUBcz+n7lMI9dXjwQxtCrXzCcQ95YlSZSBVEYjjFk65W43AJN0/VzsUVUtSBUIilRnoNikNIccS3+MQr2uMv0NVXydNCfIu21EKVyutu9uDLsDGyiWHoG+Z+peQlkkAhOrmVAmi0KNZ7MbhHhJn/N9sn5+M1RtdaWl6toDtw6V9QlExa5/xyMFDiah7Il5JXCYVv+UUz42fxBZ//CGI2PTGw11WLFbgUuMYcUC3Q7/Jsy/OYNTD2izHu7eZiJD5XfAX3sWygyHN4GP7XpeOcJaBlII/m45j4+AAowY2vg6dIcPydgzEEVpLn8zzYXghzlJv961M8EevGj1/a6ypaNT2i/K3MN/Dm7JsazuktEprYfv4pLiza2Kj1S7YNWu4dMkONQHNp1GB5Z9ysMEDHcDLdNsusxOj9Zqbu5dzhdGYmVXZ319j2S7HNK1pZskvZnUjWn9ZcvDh5K83DglFOgiWBPA7CyHpJXgTFG5BpQ1y7uHXbzMuI1kxVOt4I6uPanQscXAnWamH0MSYXWxZg6SBq0RHv3LVHNNAB2w3zn3qmY8lPXzfJ+qt73UP9dTl8Gf1nWSr5z/FVYDOBC0BjiNawIuq27qBnE+hOkoGltoyXxvFRzm3Sx8x3u6SMnSveeCoka4o6dqWOLk1zYlq8WL9ffTimdWKx0yC5tq3hdXngFdNi/ig4XT8b5WrJz1ETh54v5Vw732Llsa1tJ1pJXPGrJLhsp8SANklxgazcuKO9OQIH+UyY1KEdcAkW4dp2GDIl9Sx9Cic3kMEg1XwyjySCpxqBKS9V1ih+xktQM459jNe4w3HtxEBoF/X1g+sf47przNJEX69AB1STwThqM071oJTw/Iuft3rDG+XgFx9N3no974jTdSzjTDsSqJR5w2LY+emVDiHT8wUqZ2KJKZ+Q17r4J63sRpD/7t+r2/NPSiTEshUNwaVhj1CsCjx6Rd5uR1q8JyUZx5bGhvUG4pWlwjWY4OBDiAcU7keI/WdM+TdpzOEWiNyiNpIAgtg53Qsf4eZ0vQHjvchcONOTFUp+no8fNc9ObSbyzqsUON/YqDJ5ew+UxMgXrbZrgO8MmetbbRbg+xx49eJ4AHmoyGkVOPbXRAAHgjQ4+ZtsXZrgYIsYv6EXm+Ci7Yf0DAeh+caPJK6NUPI9fNeXX82ohTn6gpNGx71tSyNUcfCPm+JFsir/vyXg3AhhyAy+RfylVTU9jVDojjgwFXeSBj8Bml/aGsxcbyG79ym13cdM17U4U74vresrkMW+PKhvtoqEWCXqQNiueUwneKyKTbSFuuv+Hm+kU/0A4c0/cV96tpejUSm1kJILdMDadKWoup/cw4bjp+iZ4Os+JVQBA27R3Lq8TTWlYwSha/USDOSCmCaQMT52t0qXevUlEVf5SehEpS9wi17pzRFFKS+bN63PcXc222aHL0mE6iEipldrCyxBcS3jDkomnJppOR3lL80wxUcXcf4yd61AngGCGqR1INESvYikTnV1DEv7/G+qChzN4gt+5Hy2LrgPBzDLivszkw3C/1A/SIwvtnUnyhtk3v51yA95wsp/EDAii/o1ki8jrs2yCcXnxSA8VjfKAGgSHwT6SBvypoKNiOVvgCAxaR+UjQULlgFPTV0/euHrsCtjJAe7dIelhQcPuDnc+ap3KKnReBjCa+ZnkUan4IotZwjQ2+98iZgwYolNM+gqruYpG8a2njZuF4q/ojOP5VdP8/ZgNJa9S9GUFO9kkjE0YAw==", + │ "page_age": "April 8, 2026", + │ "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + │ "type": "web_search_result", + │ "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + │ }, + │ { + │ "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOsx5Tp5K7G9GIvHGhoMy6N7ZkBLJThQR0ruIjDsFF90FUjP7sQFJ4SjI/D6vnTbK0ZLbFaTusrK8rN4fEgISxzDpMf9rOPTxzHmmHsqhxGpUAjR/oWzZxmvaC8lkJRod/1AW82W8Su45qTLo+bmNIgNo7xlwwhhPkGkkc/u37uU6FJXa7yz6464z3DtiVAbO7ICKjLfCfCi4o9koVjIm3Qjuy4lqR2QztpNJyx9dWxal9l3OGbmyqCPqsenevRI+tAYtJbGs2RKr3gJSw6wXaSKvtVwBm4UmRxTg4lcLFwlZbJrdCeics109gQMYNAeZCAOY71z05yUSb002fXOzmC+G4jv+gJpKeDdP+Gn/UZvXMCbQYi3lu7oVd+tyc8XSTwR+HkdUY7OqxhWdBlJZpRRtS6zj3ukSqsZ9o2oOz9OtcwTbP4BXebZ1+d/XbUOsYbQmxg6/aHrX7XrWzRYYhGqH3an4duSCEz+mNiENXb605+Qm/aoKgfVV4sHSE57mEbX4s8j4pSNE+GU6G68+N4pG+KaEouAGZZ/SqXXfAZ51sn6ER8IOPSrllC9jBX7n0rKdFsUWUQNmqc0Sz/qYKdwsVh9EOCoUDua8ptH/1SV7hTQxu8Ehg5TKx6ClaExvVtKCiEoJhlO6S79HTyOPLcEu1Mig4lTUvPsXfG2vgYz7QIjCRnDtvgJ7/oJvmPpXF5BjUSNEvioXckFAK306HOIfbumCpmwrU6Mr7TGWdcuIUgUv3YYqBcJSXorAzI+ZvVmO7JVPkajTjKrxuyPDiE1qCRL+7v75OCZ/Il09ZeE9f6WwxDklHexBoSc6aResI2HaH1PO2emPYdcZZPUm85ONqzjh4IRboa3aZXxQNJkldjZ0drXd3nw03t48vtN1tAFK6X3C3pdIP8r9jiN/wtD9r6hDYrZoD+OUbgUuH+tciDr5sb5yXFQaPLCtvj9BXu+OX5aQTxjBM6lIfqxo8zfBMbvbti07eMnYudrkOyuqqUzujdVcWQjdjs6a5wxg9M+uuVoxBGmebk5p+D5gMGKbMQnqwGaeOdc9l8SU97DDsNCt6JnMy93o0ISiuLPcXSOCp5ZKk02+AafidbdhPbYKn9gh/UE7lDzIRJI+kLQvqYIAu7PeBqsa+Po1chyaSypC0JEkMtIkd1TE5KAl03yAHEkfefRB0JG1WnVN21p+eMpjYp55l5NSxOWgkSkU7kzw/h/nij4JlFpeO+U4U8E8KdZZWNOvjGQHs8FCTZ5GgQ/vftA1LlLLWL3bmmgMcHuq7tlH9Uxh1YJ2APWzU4IRC03oK1Q8BbL4AY3zs+EIHW7Ny42oyklRBsOi0B9N0TNCkZO1OSZVkgh3WzIxFVf5y2J7Aeim7HoZo2aqGgEe83NH/hWgccDEbZ9fW1kHf7BcsueJpPTt2/tj2cXt16ZHrZOTPid8R/lxBjmW4MZhv7I8ZhCup+RJIQlzXHsIJRvrl/06sDiunl8qpZl0l9fmb3hhoxbNyvSe260KRPZ6rzTBOLsaIqLWwCsrq7g2S6Iu1dlyGDgDdNqSFaKdTwroAJg4EeMCBWIroBLiEEoxhCILEhitrQI5887ivm1HvlaZOgwTGA4gLuzTfehwXUHmBXc/ouICWefQ8rK0u0il4a7IpVTdo4NZ4yObO2MytLtEuFD4AvK2Zdrwy4Y0Hn+6WMD/EKE/RmvFher7yplT5JUyjuL8TxNmaCXiHG8tEewy0TJOhHKfWy+iyfAzlfr619RoWMR+RaupbmQAnzCE95yYicGC8sJFrMsOEy9X63IbTerNe8ZRzoTOvczLSSz50+y+AiHcUolJuwVynWi59VOuqM/UHBdMV/LM538uYC78xIB8wS3056NFLjc7NdROBG6SpNKIefeufTdd/tKEBd7sxQWoM1MRypm4fgQl9G0GZgNca7YiUIGS8XtlvEeEzpj0t7NYXV6FbtP1DcG3O5FD6a3bkXP846DO0I1Db17NGGsBLn8UXMLM88P4C+l9t3LBHlvcSSYvjv9Wm3DvqXoGoJlaqUeRnqiByQikqx6sUkoZk9LKcpZcvwPQv9+aircSVRo7DT0nTjrsKtOZioHyi27YvdfcH1lU5V5ID8xd/pFSqK2kmaRfWxz4Lwh91mt7siT6ZXpPdOs5DRtelK/I90nBys4MYvRgrqCdTV1nHS/1D71gdsmgCCQZFTGSABRW4hi19V+xe420IoTpIqeFsHwswhDGa7OYJee9l3zaDganbUUWGQFnXaJf++sBHo50HBhi4NpX4+loI9dK+ZLUvEr3JsvEG8gQAksE1+T6FjABt70hEOoj4ByNnwoZhcm5Gv8XI5YSxRD3hnnXp1VSZbBwAArghMGzmf9yEoCG5dRC0aCp8X3BAqZL/rEcNpvietZhEuRMAWi1mHEuCv2sigC9v0EsuOUI0eR3ZHTiNBL/Nn6JLeOVTz2PnkhE2R8xsy5fXRS+pdV8BpcCmInLzB7k2ir8dYkrU8Lm0qasI5P5+EQScPRySRTQYnZmf5vqD4i/WGXovrvQs0L2YBDRaD+dGnZdLlIrUtiXP5+yeXzLgDvxpYEY73Zu+0oVyFUSe5q0khRkV1gc2gabV8GDaoXulpRXUvB+28pn7VTEGh00tSJ9tl9YaKMKYrI0cAfr2jn7WQ0VvJSA1sExT89M9HivnO0PWH23qRNYp6C9mbZKQKdEUen6TCwbeocWn8mojSR27WwUPDD4xtOWF2ugUXIOD2OVftXMDgBNTssVEa/dtPp0GJw++0sybQZHawsWMWd31GtlIa9ytSYo5E2Om8uLwFmxvqhKscAV0Md1tF2RFOx8+VJAdah0YeISiSjh9cEn3j9x9O5P4mVazy+/9o9AmBdl9rZEZGtz5QRivwECvARopg5J+z1oo/IAKpRtCu27hVzuvGahv8O0+HH2xUdGjqjkgoFU8me38oxP+/huqjXdcwaBn8r5+2k3HmZODm3B/9Gi6kj8BF6DDsQYhZPdTuw3O4Ye5oACX0WYZKEIxgD", + │ "page_age": "4 days ago", + │ "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + │ "type": "web_search_result", + │ "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + │ }, + │ { + │ "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAhDgusGthRwQ8gEaBoMZsaGfUAPQuC7mSXgIjClE7L0D6mlu69L1BaEeODnCXVcg0QugJ8eEpHtka+IXOLsFlTI7CbRCO2Q6lw7q58qjSDiF2QiANQPGof+sQVgoScXW2jQO4PgnDyRxEoRca5LyJY6DP8LVP5dZfyFuUTMSPb0QhMRZste5Hv6hJtoONNmcptRuWEpeWHy6BRxDC/E7YWoHn1eu/hdKrwO9VoncRrVAD6NNSOGkqc+rvjE4XNV+++1ZSL66sWqtUmi3P+ukLYY8Q+B3ME86x+XJC5jGmeHwVSz545X1iJ+nH4eXnsd2w+Q4OL56VGtvzZ0JC6sMJ0zY9VQvop9GSJGODdvIrqykgLAN09z6qbphbDnCjqP79UDn14EYBA7/Mp+p5aCAan9S1wFzeV1DLZZhrP62cI96TB5HTvSt+UttkVVerlTTTEe3AYZoYxGMxIMR4F2LHLbySeCCvIp/jYnj36tVRR0IzTsLGF2ijnGzS+wSkRg5qeirxMz6usrtMYgmVWaqTkpuFBbhC9Blh+M6gRQD/YBqvX1j3hvik2pJ+wg0gaMYBHjRtGY7sI+cTUltMSdP+ShtBMc653jw59fVhhm9sV2GQuF3rzVo48C6gWCElc/twGAhNAp0DrjvVgZuu8DL7ejDxTuK95cMjBWPhLmGh2Kj3SUbmLPDPv3rffWMGeX8mBq65ks85fOZkx53JBTwnUGJ8ONsOrqhP5+gfkscv7mvtylqbie/3DZJZ30N6U77xrmLjTiQ6rf6Z0bYQHIiYIZeSwm5OGq6hHbIeiZpgaPXgwEgc5Us1v8ZO39QV4Jxegv6BDNaPYvVjAybS++8VDao8lPfJfB9fNUEnR82qxPU+A4GhdQYxcyYAfq+Uot4vEffUxXrSkTHl1LxF7W0IqHFvecv9Qu3t1/spXqNpUVGJLGCfUbu9Dy0vDLEJLvWD6VY2AcUiTN/bbUIab+5uHLjTm9qJ3inwWYpfoxQaDoMsKDFjTGJEBFff6ky4PeS6BPUAOg6FFi/XYAbzcre8IzyaJeSV3MRbniC9GYaeLYqMwsn5M3ZK6T3vb5hKROlEJLlTEyQw2VjVfjrwjNWcoJatISqBtZP3fwXzg5o3xCNlQA6O17pOEWbPd0x5jecsHF9xn7VlYGj34SNmJFhbN37LJ/4DfrzyON70WFGx6ZDfUwgcrZHMS5yBLsW7uWd9QcRo9gmU3yZZlCZlV4mrhthnF7Y2rTDNa46u+As6WlQcW8d/HOEg2hlEFhwh2POsCve/GJmdqdrzGSVXz6PHqg2U3fps5TGX0PVII+rajDaw9/tuYUT5EBmUEq0z4Su6rrHMQkn1novlPapFVslvlIW1ipdPBCkf9lL87oHhcsg9DK49fD8AMfK3TUe8/LD0gQQHNifUh4RoFa27NNIjvzxfArbssF4luVSDVLwNayGvMAH9MMDWAeZc6v/TMzg0Hp1L6iJItoM9b9ZO51ptV4KUiukkMSJezFc1ZKUgL3NSXduHKngMUpGdKkQjR+TtvBBlED1m/3sTXlSHlck8mY3C3JujI3yFfQgdNXTD/zww7UFl6eF73/U5QvC5WHDec4I5Tj7mUDLtmQC2mnDrGuxIJVHmz7gTuWu+q6ixAiOHuHw1wGfjd/GjLCBePwF1Umbdlp+cg2b99nLsGWdqgV0dfMGd31G0SJI/WqCMty9L/uOCILRWz2zWa4hRc0aNVZIK3x37AX7EL+3v6oL/69ZV/EpVaXnbOSmuRpwE9U+Kyp31CGiVNYBPjLjvJ7D/5d26gEdgSuJl9uNbDYEBd4Quhnufz+Ue3hGOdHKt/34KmUQEm/qJUTu4/Zw7r1dQ6Co93CUA6XU1ZsNT5LYzV8LLZAT40CDVgm54M/NcmWAahX9XrYc5CBimG3EfeI94lA1zT/NuqzYbx4bQoCNtvSIDLT7F3EvtMMJxsc+zsUWiZYIhQSXA+3CIiX0xzZSIAtKUE9Jgpk1HLMXCS5UV9VRKshkofam9EaeODhRaFzL7+HCue+BC8bqIq+2R/4IyXPihr/5nnfCryIXfhO37EXJx/VsXCBUQsgeWJZjpRdsHCpK3nDkYjpi8g3DfjHXiOSWg9qO4hVG82FUU9oNRUOjiINd6CqMLBfOPPSm8Dipa3sQ9EYpJMDjDTtmQR6b9yfBLI4bUlUaOWHedSDdFSE4/1L2mn2xbWlBCfOFoE2ls/e5fhoGm6dThZ1SFFll1oZq6aI/pdxAlUmbklcf3Shh4j3fLmOzxncC6zr/tLDWOGln2jgx64EeN7WF52ErzN8q//I3ADpkV0qAk5yPL5YX4FkOeSsyViqZJhUSy6wIuIPKVuHb2vONeMAhwTJbfGAUfDIW7VgUc8MDV9QBH64Jh6jkN3rLdOtMb3nm8X4lCg9xpRNxf6B2ZdzNdYj/3Ni09iUaQacQTcmgr/aeXI9V4lUnt+eeHzKX8rGUhvr85rkbiDB5CcPulxXnVQQRlfRzqNZANjCCNDcWn89iSMILf8uc6BqkO19gCKK2aPdQmdPCng8qtJvr/LH35LYr6Hxn3AvUHESwYRPjKpMGNkpFQbYy4BvTjMAijFw9BhbMUvkeYEkX35wIvLH5ridBoZawP6WxrlbJ7WKr0jFwCBgS6kbvvRR6+Ixhxhk8VReoSdFUv3bNSjSDvP/bEQkM5Jutxl1oHU8S0ISSezER6IU45UpcuXMKqOVLf1Lh0TUThZc9FArrcJWvnYaSYUwr4pggqJs64NnOInq8HSa0hWsfouI7jxff7NN9jta8ss0pzL3zeKkLIbTCOFNfTubIKuxDxONs9nOQkdIofvD4jLJolisfGm2zNKMKTO3TX4/xhfW9ceMuEyfcP6xrPlOmdhpdm/xwuP9WM0yp7xNw/5H77MznKJ5la5gmB0oJl9lGQMkb7UYWcGshxrjXg6voJl3aKEwh5s3n4+C62+AP5IneBvbpyAJ89zLIzuR0xCe6QYE4IBnRfqg+GDHnBNHeSFGsp5ygT1T8t1WmwP/mCaZM1XJ50dznsvgxnwgAC0LjNYxRmPUZx+3R33yIxtymVMHcaHJpoPtobqE6kVV/LprICwF5hj3Vg8CZ6gzT5ln0JFZUyzUdA4FHoZ3+twLQ6wJzVED09ybTYmivw1ci0siMl5ronuNL06cNzQ1ITHIwKbOS1d9DnhnMnX7xzmEIUE2vnlBbu6LusaczySR1uC4AvcScLvPjGs/m3HZhVB/cXsDuJ1Pk0nhOJK/dEZ8XzDsHjemTUahqZbvtGKSBrbDjjYbYGT/7WTY6D4UgJEr7nV3t5MNMYJsfT88hCokUxJphOHL1Ezs4uNso6NNoiCCjTYKvNKW2drae+TRo6vAgOIxH1jw6l1Kh5J8tofJ70iqgssCUPKA1EJ1s+RWdfgYyA/8oYoc31GPuudkEiauaagr20Y4AjojqbwT9VzxAqOVDqFB9gha/D1bcl9uOjeWqPCy5bZCxeua+Krz/3+s8KYv8hL2CVvNmey5otWqXpsNRBPnnSMt13nowXXAvKxHmn8AYh8zJPEnW315rFsSwD9gAKitGlkVqLzF5DNk3KkY+n+JVheQKhqAmR8MWwwIhCUsPHt+eG3mkujwihBTuQWVn5Tkb8X1Xeooys0dh6w+Szc+sZzPZeGgIL5xz/SWHiZxEfEpA3TJbwDiD4uaJNzsxsiGbkhD3heuiROwEyg/q1En3Vih4OQ431se5J+jvEsO0c2Mn0kUkZmqEW3mQBHVnUwBgYVC/Zi/ib8YQ8q8ta/ULTlQapwaVozshFF8T0QdMXnoIQz+1Z8rWILY500M0+C1tcd8Q5rJOYIgMlVNrrOzHR40uj+Ng0U5f56g7Gs3hNoGg+Vk8R82yUNfdGngS6Q+BoKkppGrcoWyggCmztYicb5vckykdRdXkxocNtpfUkmK7RxCXSS8q84HLhory/JIeBCR+KpuZSGf5S9p8o9qpUvP8Xati7Y30wjFLDHEUXfHiV0v/3T/MBGViTedcXXyVO7TSF3w5c2sKbQ7kGzQXbkCYUcdWxZgTOaI/FwUowC4SYQwChVn7qYbD5bvAxCfBe1ENJRy6hG7VYJIkZQ07Bam94cuGaL2ryxEwJy2+3P4aOwnumSmuPi/cQOBFHMjXRzAE2lbSvrTEXW/T5DCYOFwCbNRH3OSMwTmCsstHedVkqiQvy6fGRx3tZH46DV5VF1s6Ya8UYYwlK+mLB7B5jDKl4sngIYcmDou83TYCBOdQ4oKpfIwrAWybOahi7p/JoaUHXLpKH5hRgRRKOqk6gMpCR6PV1nO42L4TapP0Uo1qa7z/BuLyg53te2JCiGK12D8AsSohYLKJ5DZsPId0J4+oyiOg+gTbdx+cqNRvFerbm+Lb+3+jfBJwT19l6xSRpyiDG7jPr5QYNoR9q1K+qqdR8CE4IXWMwn8vQoVDTx9iaiW9MpyuKidSzNPAXdQwZfbgHYl8cEGhzO69WZ0KzTH5n4A7w7g4xzs2iQlbprxOhYU5ajNOccKN/d3+A3dXmJXsrZjZ6qkQq17UYgOhxo7SkbvNLBn2ZhLU+krM1jVZed20qAIYDWMxNY2OSP48/NYjY4wqaDeXYzIoIgnoGWWlu12W5FJ1bx8gErEAYBGr5lBIs9Bv6ST7Yhpee6BynoA01L6WpzOW4FrquoTCR2VuE0MpY1GJLuy+TzI4lsj+OZIvA1LmastW3UavIP4mL5X5lZ1vxfnAxjuX5LaD4SwBc1TezbHZ4t/jhoWGmqSn0MdIG7m3sEJo2Rc31bsaFi+gzksHKp+WyydIdMC6zlX8AGXG+78KbSJ+4gO6eAPhX42CZ/o4dvmwi7VINcCnh+cAesRX7U1hL2vXo0+wLv5HiCKO6KemrIoVzNnaAx6qFPAHgR4t1x3MwwMmTw9UWoVoY/JUtIPsfeo788u7q0rZPI7v9KByv8VNLsUPKegJcsbC3eqVyhJhfB6P2TKbzZJh2gaaIsLKv37ECBj/9bwzVRz46RigEEO3VWVxTZd+ikmiUHJtcapROsClRZBvKUzXESgt64gHILoX/Nm4VoqTOTnqNghHb+kLoXb+jlbKil+9c9ogNWUtdj96opHI1nE6rz8Y73eQZijKx0BeQ0TrOlCR0lLZ/94FeIYuj++gxdtTMzDBocyLph3xLdXSyGXBgG0UXI3IaBmSGSMJxUELf6lcN32kgqb0pbsEVOxdATPwMKAQuni44vc3PCRp7aoH1kBXPEMR90ynXXpPQUloMPS42O4P6cAJCw9WvC+3S2aJarWWjgBwiP9uYgtnUdEF4z4KFP+138jUaBwjCMsYshHdXnBY+zjNpUxufqkg3p9YA/GYQ8HKyxj63sOmc9SNuOzCuIg8+J5OgtfJ1GHMp5WBcRaGjmG3MbYGm7bfxouklPKL+jfaluOrm71s1exVZGe6xQs0tu+nwFjOmdBJyvpu+gY0v5z0qTibhFA9w8IACpDD6uEVO0DG04ES0w/RzkJCkyHQnCqphXusn8uLC1HZRBSytCREr/24+r/afcC9iaBI5EzCyZU7/jfMGby4aYMfJK1Y51R6nqUmL+stjT3PhgD", + │ "page_age": null, + │ "title": "AI SDK 5 - Vercel", + │ "type": "web_search_result", + │ "url": "https://vercel.com/blog/ai-sdk-5" + │ }, + │ { + │ "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA+bo5DECQ7RbnLGjRoMu/V2rXAsOajX63HXIjAnib7omq+dgSs2/3/GJneFB2vtjsOGs0cqD1py6dBQJhycEmg/XrQ2nb3qrnvk440qyg34BcSaIUyo5Umh5MLJr6w8LfiGMmjiSeYGGCiV+8N9DwtPuR/nTjT5xDBab/uarBOhmUvTps2l5pN+JWCJDsqn3mnMhPlssaTH0kNq/OdeUx0B/1DHfdO2j9QkNYVSIw2mpXexPOow+f6GLfpjxXSq69FVQZ7qoXRxktSXrubyrWF7v+/PKsuRJcip5X0uY5+ioFX6dDWmz4qPOcYL+XjUqMGaWwWggxUFGue2rG2szCeMBfo2jupRsAEeQBy0ppt2AnFjIQwHIn56fAfVB3VtGHQY8UMfD7FA323POnPg1JncaZlEPlarbAZLXwW9VoU2s6Xz+wtxkbSdR1RprOMcjfW6Bnrpm6sCGIlYmvRkJpKvqu68HauZHgOC6J6xZr0yoBP6PUoXvDXIjf6eYnjupPan3oOd2RsgKPHBTEgUFNBtM8uo3hnvnPrcjU4UVHvjcD9i0mh+WeCblHu25oCsImCVIt65nMHlB3I+imPP9oxphljgp6bEtRpkCXlWCNZ+HlT1SeH3ABBa4M89Jj5sygm8hSUXoVDP4URQQGt1G161Qd6q06T+wWPLhng64ECFtneLbOuO5ft5Rkc0FfG96zYzCKhk303NJcBxiVnvX9S58XRFDFVQCgztiDmnpCwppU15XBwUZtuOdvhgGWtiqBiWPnyGEVE9O7Hf+nZ0AHZ6PEXqdhiIJ4Sgy3RcyXS537cyxY0hzIFvjElQx9hHFsabnD5lHSk7bGpnGrKHguikArdLQjSLvyjX0qt+VO5mtmfPmvbCXdt4zM4paJMzuuaGu8sFornrq+OPZEuqBA72lALjYHBKDExGwBqw0YZgE3rITVJcwiYWuf3u7a1Y6cjSlsDy6r4kq7riC4bKLBLG3Lkmh0uOVrjLlvYFatsBSGmhOOTVb8hVQnU+/lbuzJUDJPjktXNjElYTFDfJARyCmeOQJsPJCSmCkk1CXjtOmF0oERg6v36rqMoawUeZ2z15lXfvGH30KrW7m5ss1XQ1l9dvnNCfvd96DAOa63YCP8HBQjLt78C/F7hDTtPvloQDIdAGJmtwRWCr1o7VYnNyhpP6j4yrRIlkkMq6DQTETS26+NdpjP/v09Glde5UrDN2V1V+wK0LK+TBIJLb6QB59GgMLStBhxwf1pMsOT2Rg7tkJTesfswcJQt67+3PFLy/FM46tSd712LLKM9u2nfcQ7M30uVVy0LXeZmROmdaNMiaIqE5G0WLynZAQlyKgF0Gyqofiq0adi22pVB5tYoRP32nHy1Lxae6OaXlo+Hb1P4pZldi2OtO52olOC6yurCajDpOWHzwN9ji7cMGWCyp5nWIK+tzZ0dHcHwrKjSH+2TK7acgVr5HuxM51GDUVhvXwUsgXAHwWxeFVgG9yDB04uxtqxhHtX9Oyc/R59T6XZYhxmcF/zxo6w6syY1svn9XckEfuIsrDDiFQrxV7Ns6w/KX4su2+AZM5uErdV53pZyW5mA6aUz1beEwRlKNSonRg8Y3ZG00B2KAmX91FR0B24pWp32BUz0rvf9RFwdjnIZz33MnErLlAZTLDRM9Gfk5Le1/4HW4nU0Zesrdwh6j0DGkgHPbi0aAbpkPT8GgxQ1GvpNR7q14o3N6/zkiX5YT1c7jpSfO/NHNDGNfd14/Sg7LKTd8cWorKmL7h7rXPnJIKcnvQIB9Qi211yhBCCdDbbvnMFkWFWXzkIuzyaZAZAoY0dJeXcIxBqR+6RytG7jYgKuoHOgMDuVoBJ5QnOKeI+qEWiKu9TLSpAUijwfK1UzEbix9HA5ZAu+hLPAg28Il1HqXOusilIYoU6N1IZaB86owGfFBxiTPeud+H/aVmlaHStBc48vjMJyBH7klapsZzx3nA/dVulbFREkX0PR0xpWb5+6qK8pwDlg+swo566y3OzwP3lrvzBklnvhWcqKzijUJismbu+9KhNJY9bGT/JvRwZc8P8dhVlpvEx7TyyYjqY29nvUddyhDlb44M2Hs8I/qy4Js0xwbGlI+mQCrzXlF15SrZozKW/Swwr4R1CgMM9TeY5nutgSHOxgMSlmq8yjJfvmuOtQeX7c7bqZ0b/OqwESltmA7mM2VWOJ8JagLfkNnfTKWL+GLEyC6EepZyFKbK6MShrKh3cb7j4t8pOvAm1fz1pB17nltbsSHe2soaQcMDY04+/UfYVOb5kNAWQOgCNseQi51/5GtJ2UXkNi3F6nc0Kqz6WP729ow6GXGqKD2cnwGtBl9VmhpWIBirRHTpVW5YZTphcWyRI+aQ5711ajDgY1bz3BSIe1nC8KHwi2JtPkoL5Z/KFiVGqYQhArBlxqvGAM=", + │ "page_age": "3 weeks ago", + │ "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + │ "type": "web_search_result", + │ "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + │ }, + │ { + │ "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPOtDoyTBsMCov59UhoMPbunuAMEYIXojwuQIjDJZ0FIl+rZ+0pOp0C9/n+pYtV1i4BKd0hQNPSuCDxBQ/QgIjdvHJhJ0/KYGjFNPhgqtRAqbZmw9eMNiHfH2JEQtnSgG5pzvo7S+2J5x/sqrX8bGQ1KIwJsRLxGiaHUxpVofrQ34HYra7TSUZOimRjoA0mpPjd/jPiTLjpqhBNaZqkHcnxYDpsuxeC7acZ+a/nSyGNCP4mY//eml/nsfwiB6qltseN5unYDKM0onfWk9D3cUU0k8SIOfDk+U803Md33D+e3QZz3Bb8O9XO1KC4s4JnypnYLaHatx3ZH8F6C70v/tJn+cj/qtQWOUOI+L3lbqVgFA/ykhzqkPojn1YWOtAjHTeidOvk8KKH7EMPv/DIfOJz360SF5WHxHOnYxe+aozj9ZU1hIyvsimuNao0Wx8iVGmlSAxtfTEoSDo2ohlQpELSlXraLXZWAYKJwt9KfZ8/CpauDEf55ioSwuplIxuaktAi2TBOYi7cs/GvdVfSYS8F6PAcF3gSaqNVxWp3sIDyW9hqzsHRP9YAf+vBVuEDbfkouEDIvpArRxhxWkN7QZ9tuPXIbG0QK+LpezjnjDhbLE59dAG8cq8gvPutkuIsz+XKnlO3C+uF1/BSExOJMYDHh9OacnrUUYATmkt6CblpTJswCuEuUecP0wAQ4QJV6wbk7d+kYjbKYTUPkwlX/kRoKTg0a0+bhy8UkHJfRMjNNXPsCHc2sLBslvlkSiwJNgfbzpVDZrDg0C1YY6xlfY4+z+cuhUVl81zsChcEFhzS+7nsNk2rs78utc+BffYgJcu8ZsA5c2AQrJ8ciDIJ6jnxscL5liSd3wMdrnSPZRGb69QvgR+rwamFZLZz9XrUGzuaDwS+VqbTThytVtAKJ00mU4wa5vl2pKOyqmc4BU+bebOPVOUdjfsxJh/ToSVsPafN4nS7LDWeALZHA8gd7N5TBEcJkUNlWI363S+PkqO62D6bnoWTn9c0Tg33CKxbXfKbmLimQVlruGHYtFFEl+U0iunu6UMUZ35+UAJ7ECXGlgcoeo6wB3qj3ckh6B46CQpTV6MzemmC/Q1N+NGgQ7MydHUHVVLTqEcMILaST9wAzoijIX/fxRLVg/7L++9aRug5G2wyx10+t4AIvk9Gf3OD9vTVTGjA+WVZ3Y4R52YVO6DlbouhqvppP2sdhdWx3KugpRi/zD7cPbLWt4Xx731hp2dB+f0v2x5tW4w+24U60zGEMZeljfmP0CkEEDEMjko3Z9K2fJn3rRV/kgHTrAUvmCE8GP2v7N/2+G0rjjn1M29zHSde9uMmmlbJ0/LpG5G+tvtjno+5drCcGx80M7cji4o3TK1ZHK4f4b4j0MCTRGKWQ7G67BLbo3+qTktFWPti4dn5PcfyHRSowq8l0wxLGR7DTRqBOS/Q1gQjl0nGHXCWw1eKaNHZC2ttftPVb9dBxf1Bdx7sXn56jcig2YsQrvI10AVyjowhCqCTfWm4p2QtZ6PqKFoJzdHj5oBCQbqcewD8VlUZDqBG61v3NXfWN/EFZTjQVseGU1CGWGcmoVd9pV83EaxjM6cDaDMBkWjcdgdyssWpu42q7iVr1+zIItQ1aJKSuQvjRlc35TtW54pLNM6uKxkd3m2NT7YLcIl3DeUqfMx3na1aw2cKTer51ekj12o8YPMSrfRtltN3rVHmtFNCNNv6WXrvc86O9A3dmW5ileszqV3GP/aA5VhtjvU+wQ6fkZStRMAxUsEVKtVzeS/Mgn9gT0oLlbIx8tcXLaFm7W4g+ASgUFG6Y6SYvPdHdf4WUscvR4LNl0QTmui7T7qvehchfRJk3hfbdxp2WmXiInKZwRr/ZoH6sWa32NtosURkR0YDmCIOjSfa2q95zhl9yy+qaMXMlbTfNBwW0eKWKV8W/rDOvSfqmvEDW3T9x4g3nNMVyPos5I/4J6Rwte+fxKdcbrvwH2N1P/KeYpBGW8JyJQL2ETHYqgyFEkUPVJnE/EVjwMHNYyikUkOk9EHjixk0cS16u0H3JJAwtieUz7Ci0TG2kzQMnULo21/I//snJxwEUpFX2RkUgcjburInDjeFS3vXY1Ew1a3Dixtob0QFI8NtVDMYNERAWH5DKSqkVLYghO7ABgERcqodtpXOwNBxRS7J5beRY5M69U8Do7ex2du3uiEYY+M/aPZrr1uvVb9xiVzt2g7TBDpgUyg+Kvy+kDRVevgggX8W3TBbSc2Ow/ESOhC/zKzMgam9+t/K5uqe9uCv63kqXECXfH4pxhwY2E/n8MfMn2o3HlV2m/2/XIUHLoo/YMvLi76NN6sO/4NrVzKVyptR19yZ/db0QVWkNYfGDx+mzUae968yUy+27wXWzLu+LHtFIQJfEJk/n3NEVZ8Y29kaQGvWz199X+BS2F7n30bd1suDU1O78+rGIsPa6LrahkECqgKOOpvhQ55bIVBBnQ8zFzphFPnoNlShPANjK3K0ifotjUeclyzT3DVFCGugVglRHVo73uxWUX/UbRVyj0r3ervXDiB5WC1xGlvPpQVtjAqSeiX88SYMXYT3MnaIk4zcbROtjuCoKUbPpaix54PDBjZFK7be7iGSdFfKiUMFx4N42pSzrJNM0N00DR1AlG25o9eHN85SH8Z9RF2P2HI5U9nyUf8Ij8pxZBCWqdexT1T9+eDd7VZAum9BgAq0j0ka2wXL8i6w9H1VGQOTJZnm5tzjWIX36KgZqgr/sT07a2I9FXZv/i8vVm/7NjfIskgp5dWudfDyMJGCI18etdOhC29r7XcB3lWYhPwaqynnZW2Wq6YBzWdmwE1Dw4p9ZJRW0okRfpIeyVq2YTTy82divlmT3bA6TlaxpKTiEP4UWCuCy/lnvJ9F0jlLm/Ig6lR45XKbuGAM=", + │ "page_age": "1 week ago", + │ "title": "ai - npm", + │ "type": "web_search_result", + │ "url": "https://www.npmjs.com/package/ai" + │ }, + │ { + │ "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDPfDIJW1qbxKekFpQRoMg0iAQRQdFyNLsmQhIjBhm3VX9bIr6AykKY5axx4eIBZxrv7ldn7VIkWdRFUUZIuU3UYk08+Bcjv3erHpUxcqixeyUisUORxBWCE/vXzSjB3vSgIqz7Y2PFZKU6y9oiEo/tWD/H6i7HUs0tg6oLR3Uy46LwyZltsblz3A86cSYSeU2HAVU0PRqtkgP1dwyLMS4vkY9T64NpqLEbNmAWtOwQB3tbwRwYkGNC6pwv2nwhbUDVzUqMMQQSVKybETCmqJmRXlYfDg/DS7yl1RdBB5z8fBvfswEA0rmS1U8OmGwlqUpO/51CQotJOvUiC83mIczCdxoVGQynpFpZMXtDkmQbTr7IiUL16qHBHazKGKCtEKy4+7FG+JsP+LoIila13n4RVydyA6UCEdngauJ1OktmJrkQ48f1ZnXFhMlI9innFvGgYdjlNPd4EVu7IBD71dTDgQY1X2x1o+s2y3luOoy4HnMp+Ifrop2wKeHrCNDG7zXHmxkHnhJJu0xissBQLs6iUZ/FTRYPJb6mH7E0oEcaJZsbTjaNqqJMWO7qzsqQHjJlrGVVWDHMa4FSEZbpLcOsIBSK2mFC5+NK+48xBuDg4CKAGrf8vAOAKRaNiuTxw7XurmjYhC7qgoErnNPGkThQVh8z8H/Dq2j6orh9mpRTAoMmFVPzq49yWkhwUoufNM7+gRzKkVAi/XEx3A78l+1CFcHR5v3REiufKHQcJQAVkhk7ZZd4wUEu1S929y8aCRi7JlUeZ810qA6nnxXGvmmGA2KIm7S/VBOd4ATTADBQP5r1EqzRC99I/RoVJIr6A/NPIpste6mqDK72q2sQQ/C727iBCeh56+mFPq9yzWEig+dmK62N8+SWeDnrNCGHRk+KTM0Smp+oTn35J5gmONlQuyqUMh0+zfs9y7TXrpxQnkE/gCAlYCpFOU2wGqW20cQ+uD1elG15aCX5yaRIX+yzazVwFxV6JviUHSZXTWVWKddR3PgPZ+lApc/yqLaiTyBgL7JXpNI8gGRLhgOAZq+LGJFdcrn4EDRSygYZmH4itglammiA12s1zXrsxmUeoNEG9ECojlqWWjvnLTqrqL5aiYaP7MV7wGKGVjrliJfQGwBCVS7D+/gezhOxzs9hEtFknG15Z8JWlWE75pQ5jUM5Mt5+3m6cEEyqyoN3MyBjDd3+epwiPntVMpmBiCqCLNpsIpnVk5ruKvstPUMq+63EXUbJ3KNOawyT9DBtjYbmjZAQy9SrBWMVLHoQMzYJxwg5+9C8wIPdRIoHJT1clX5q0lOMl09IVCCG3EKzmgC8cXlB/ayUjLqtENvBIWd141tow7ylo0qEBinpig1leUuBwbk0CY8dJFFWpNtIAJG1qIl2ZBRoSFn1OThE9fUQzQN/qt1p3DHi35Ye1gLNRyFyAQQmxuodEY0tty5KTCmR5yj8pUpKDQ/bw4kHp5sjvWEDh8eSDOEMrD+vg8ezRLhi1lkLy6BJ6OUx4xFQM1iks9MayFXmeOcWD/ocIwcRlOi6QoaxsWlPCnVYj3TdYIBCxcYIdgtz3IiT/G1LBJzLUPCAOCkYohpGDiGbn+fAa/Z4LJ1NihJmZBz8i6LS1H6mN4TNFYtEYQIcp3j4XK4UVOBi4nmeIURbvyYoZpnfB3gXh/Am+nJxKXv/BMH/yKfNZTKawZuemsUrju//F7ofZkjjZ4MzbgzP5r+BA3ZA12K31wAPdF2GqwXaclzIb9AUCWT9Y11Am0yLwT4UZWPcgU0HYm/jLJARZry2fuwcZx4J0sIIh4yVU2zXipb1id8qkJZz3wBHii98pU1iRUpl6hnj/Xu/hY36YlLdymUQ5014tkio7+z0cKkrdGvrxGHvg7uB3DxvuWnhy9/OrgB2QQV8WOhqNL8vLXWzxwBJGVyRWt4q9RuY6S2omA1vH4BUC5XWTd5lICubWculAr7VUJnGcT0kUb4rd6C6dATyLDnGX33fZLauvaVVvOyzpy+YK8DF5+nNuYPnObISNHYpSdj/roz0fgenLWz1OtCW2uxag2LL4YGeKSmLVtuHglgSoKbw0gCDBnk6WSpyW529Pe3SLjc+zufyQvJfylQ5u6vzoEhafMDsnAZlyQP8WeupQK8uQF0ieyY/i9OXntLKmm2ehV3hpRK0rtOpX3z0YXLcCvpNIqf1R/yPWjHrNnWRu2hyJ+5QEFpTgXVBYUFrRITYZAfCIC7G7NnjzvY9wJUtqv2iH1+Z8Ynz24MBfUAGjESKdh3OFm4fJgNdimuApPMZieZVVt3x9+blLUvaBGi3hqzcgmCdFeA1G1mKjFi/CKqAMtWmVmpwGgtVyY6UPq8NpB30VhOYBCM+KAwKo4NIxFSrg5tYfWcuvCt3VOARFFxIBi2eIKoVGs8RTSv07rYbuxq0HTHpjaCQccuii7emHg2Wq+MWNg1RvYuXgeB5PnygKkNqI/51w7b08CXtyV8cL7xP9kjvO1P/z3SogFtSw4/c7PnIc2d08jDn10+U9QkIBjc4bEtH5sWTxdhifRzwZQ0ApyAHBGZ0+wMVFtUC7Ik1kBMN1NnX6Ly5K3S7PFRJozIdXGkFn7PM5YfmNFMEXT4QMVsglO9yPkZkp4CiWcrw9X0E7pEFjuPYYBKCAoHozLQOGK3acYw6EVW4iQWJATv4XNJdsUWgzcLh6yNjY8HUHa9iQKuVT250xXFlxumyDlKVgazCXBgIf6jxDx8MG+VdtOiI7xMKvTaSBi5pHDaYWERJ0JOOdMUeNpVpT+Okudpka6XqcICcrWx6SugQKsyY1Vr8yluUO1FeKhb2ucZ0Av8Xw+NKyp3etPa/TK0TU862s3RO0VQtXUsa6g/F9jxzuMDPx//nsVMB16qpOMAngOI8+KVUNsjM4a7S6AAtN0qSIC1l3tKQpLNGacbv+S/hP/pqJWipt2QqRWa0Wfo5Ql3fGw59fAEhEuHdASuuA16LOIr1HUGcAtmAxgfDZjT8DBCQqzjBSnTF/7C52xciCjzYO9gXSdr7JjaKKiR64nsBxNP6pEwOqyjYx795lt+YTGHXhDl/Pe6T8QJPD+o5Ej46evnzuKQY1mLE+WfYayGesOmHLpAzrodAtTC2PFYsumeWwn1cyJTNrCgPwvxfW5fEinBneK0P0n984ljRVSGiih4bfsE2Cul8sFwxOFFTP6reT4HMvVphFAbVXgEKxqy7uMg0WyxTYAdC8Plf8uUgPYYgHJvqRSqZ4ZaOr2WNuIi1PHCXL4EX+j/cVqmtsPZ3Ia6mr9DRIc1O8gHGP/BIhKP/y8EZ58ny2v4ns9JCYfscYzmqLht/ZnWHEw+ROQFlQKrrp7WVf2uMbaBdqMCy2DyMea1+fZC+bgVbHBEm7aIAl7BhOmtHFPzgxcZv8Fra2DCNATHZneVf5CGMHXfgK8uZG+CVp/hPB13VZA3jsoaoY9BkHfALrfbO1HTpOJH49EKOczWoyspRuIWlMiHxNz7/+QnBOaXI+xd9/3JOOh8kUIa07ZrXmGEmcdbALwl8Be8FXXJ8F7KQvGkWhhZyj9V41UBRezf510bGodUXiijNYhCgovFWSlD+9LtVzKi2qRW7m+sdSnfWa66eft2oABlkGO7DfcbUuUclWXRZqzxvbLe9vRBY+ZzN73E9JUFwFKTd57ixV5Q69/JKTJhu2PFGsXNlUBy3944VBNqY6jNUrnPM39u0nLhGDdt5/LXN9HRYXOet+Nz/RKgFjIf99J8vHeFYnPfsua1XyqwrjTs0dEb68ri2F+9y608f1PmKtEuPbm7/Ofn3wfnZRh+iJfKaOCuP53yTVPsEPcXa83IaAaH03EuDPqQhfD7batO343Py8UMgRzSxBOCM9DDc5wqnY/wlIrz31vrpRlHwmMx+FXie0PR1IrsAVAtKWBthwfIQgVJd7EzZTcjyn6tJPiH+C3Upx2OE3YhxN6czkZ39wrt5yLl+KTRfDUnnkb6pXL5P9NaFQ/JODOSEE0iKFVVH7lm7k4jXEL2kiLbjmiVIqtAdCvJsOz9XpaU+Mqzg6qGD0YAw==", + │ "page_age": "1 week ago", + │ "title": "Google Cloud release notes | Google Cloud Documentation", + │ "type": "web_search_result", + │ "url": "https://docs.cloud.google.com/release-notes" + │ } + │ ], + │ "tool_use_id": "srvtoolu_01Ra46kxprvPzeWyWomhLzvK", + │ "type": "web_search_tool_result" + │ }, + │ { + │ "citations": [ + │ { + │ "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + │ "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCthnxqf0NqGDYOyMhoMBTmtvdkdih3VOuzSIjAHJR7oA3zyLnJ+mcRNAaPAHZh36NC25j8Z62M2TjPzxVpiJBYTSR67PGhIXO42QkUqE52RCjKOVnuAi3+h0dm32SS2GEsYBA==", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result_location", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ } + │ ], + │ "text": "Vercel AI SDK released a patch update on May 13, 2026.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 256, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tool_choice": { + │ "name": "web_search", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "max_uses": 1, + │ "name": "web_search", + │ "type": "web_search_20250305" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 75, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16216, + │ "server_tool_use_web_fetch_requests": 0, + │ "server_tool_use_web_search_requests": 1, + │ "time_to_first_token": 0, + │ "tokens": 16291 + │ } + ├── anthropic-stream-thinking-operation + │ metadata: { + │ "operation": "stream-thinking", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with the number only.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "signature": "", + │ "thinking": "The user is asking for 2+2 and wants only the number as a reply.\n\n2+2 = 4\n\nI should reply with just \"4\".", + │ "type": "thinking" + │ }, + │ { + │ "text": "4", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 2048, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 1, + │ "thinking": { + │ "budget_tokens": 1024, + │ "type": "enabled" + │ } + │ } + │ metrics: { + │ "completion_tokens": 48, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 49, + │ "time_to_first_token": 0, + │ "tokens": 97 + │ } + ├── anthropic-beta-create-operation + │ metadata: { + │ "operation": "beta-create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BETA.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "BETA", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 + │ } + ├── anthropic-beta-stream-operation + │ metadata: { + │ "operation": "beta-stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + └── anthropic-beta-tool-runner-operation + metadata: { + "operation": "beta-tool-runner", + "testRunId": "" + } + └── anthropic.beta.messages.toolRunner [task] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + } + metrics: { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 607, + │ "time_to_first_token": 0, + │ "tokens": 663 + │ } + ├── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: "The weather in Paris, France is 18C and sunny." + │ metadata: { + │ "gen_ai.tool.name": "get_weather", + │ "provider": "anthropic" + │ } + │ └── get_weather.lookup + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01T5kK4zJvHrkA7kzPucq17G", + "type": "tool_result" + } + ], + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + } + metrics: { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json deleted file mode 100644 index 7b7fd46d3..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json +++ /dev/null @@ -1,605 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-thinking" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-thinking-operation", - "type": null - }, - { - "input": [ - { - "content": "What is 2+2? Reply with the number only.", - "role": "user" - } - ], - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 0, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 49, - "start": 0, - "time_to_first_token": 0, - "tokens": 0 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "thinking": "", - "type": "thinking" - }, - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 5, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "start": 0, - "time_to_first_token": 0, - "tokens": 18 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "BETA", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "beta-tool-runner" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-tool-runner-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 72, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1294, - "start": 0, - "time_to_first_token": 0, - "tokens": 1366 - }, - "name": "anthropic.beta.messages.toolRunner", - "output": { - "content": [ - { - "text": "The weather in Paris, France is 18C and sunny.", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "provider": "anthropic" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": "The weather in Paris, France is 18C and sunny.", - "type": "tool" - }, - { - "metadata": null, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "get_weather.lookup", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 607, - "start": 0, - "time_to_first_token": 0, - "tokens": 663 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - }, - { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "The weather in Paris, France is 18C and sunny.", - "tool_use_id": "", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 16, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 703 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json deleted file mode 100644 index 0acff38b6..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json +++ /dev/null @@ -1,471 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-thinking" - }, - "metric_keys": [], - "name": "anthropic-stream-thinking-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-tool-runner" - }, - "metric_keys": [], - "name": "anthropic-beta-tool-runner-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.toolRunner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "provider": "anthropic" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "get_weather.lookup", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json new file mode 100644 index 000000000..45d166602 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.json @@ -0,0 +1,896 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1415 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "anthropic-server-tool-use-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "query": "AI SDK release 2026" + }, + "name": "web_search", + "type": "server_tool_use" + }, + { + "caller": { + "type": "direct" + }, + "content": [ + { + "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDHDiukuMZ9P7cuHVNxoMqkhPqfj3Qtd901TkIjCiXbRiwj97e5IHHCAe9toqN1n26qStXg+l3wORzAAO0F418d24xozFzdqBatJOT3sqzgjrhHdfBQq/waT5zq/ue64nAX4hxc/Qf8+wGfW+EORQzjyzLectH3kgwzaC/PLqJ+SBqunhO+tBWaDt8qzYFB0Iz1TMsLkCjVk48M/aIvwSlqx7rhgCToBDqZUdKgpz2yJo17iELqY6FH7WBiSrpWNBqn5yWMTrwTp9zMA7MvjZCukQVDbxiAzpB2Ms6oL8YuhOwQB9sPbh+jrQ4zpQKJehaYp1ZhNU/osT3K7fUktioBhY7VfS3ofzSZQInKJBRSVSpxxWj85SH4GOAjSMZH1QpJVswuudAp9xQ5KSwPDEn9puSTT8tGdEVVoYHhwKfMIhskKuynAj2924l4trb05oxOOy/ZlTEsYHhcU4JdO0VDouA8kmIkOZxaVuWDQoD3q/L+JAkLk7BAYDItxvfPnMqoVzzgm/zZxF5o0G7tiKBpoZrIMlFuuiFYqQSlln7nFucjL7lJeYAiW0UcadRvbC5xglgMmc/zIPwP8pjdYKnGBtJF7d8MKs5YUXOmzj2CEW8rJHdic7ugCAn7iwEwnldPnUoEirRwFGuIeE5gc/kvcTwxzCT3J1rRtYGYuCZFXUWke9pTKhDa9HV4pBKAO7XPuXRt+UFjZGY1JGfbbx3n9tsdd0DdiEoMRInIxO7BzO+QgTkHc0w2NT99j02XpYiJU3RzGM3SMYdTtG8bdOl2Eb2dbA4TYuOuL10/eMeuLad7YHWtyXgt2Xd3flT9HyQHF+rrhEUYY6OWp7wecZFJF/2N4lZ+tWPhzcg1eUwImRU+ugmed3jceaGQH41yLhAwVHsQUs8Qouhg4zHjBfJbYiV4PKTwvZch6OaEaia2KvDFK3LKTGHdeuqqof5AuUA0AUx4hFn1zFU4EGE2pVEuCoscHhTorpB0B3onRylj30Buj46m6zJPzEt1gXMP5Zm5F+ZX4xbusV/Cx81P1lkKIKU5ebnCdjGoPIyUBkcXWx0AhQfXLLQJ1rvaY+yJrlM5VmnDEMN2yEhQlLoZqBTcHTRIYHdBw4XadWGGNiuVsOCgqNGnnBRBXDa3YuB6NT2mTZH7+geX+seSuO4ySBMr5wQt7ER3FiLkODM5As1Nm4bpg7C9HBBkj3i3RJgohwvBeDL2BsQQYxo0bPK/NQkeLpsoRpNqIgSOhGJ/ftEg3NLj8W1F9qdtK9Kezbzw9t9GhcRtOrp3rOwoZfDuCHFSDAFd8Q0TGRas2SCp6HDK8WdgIAKvV8oRbUNtFt7sE84PLIqXQlhjyw9ccsSqJc5U2rwuSd+J77Q77W9Ux/i85cMmxlXFOkyX/dOGrH/N7fOyjPSxA9lwx5r0tb4DPdAB6oOOq/UhtCHDNot28Rzy1tghv7ErlvVAQYR1b60qdI7tCynmk0aKPWN/UwHarCEn7tVsB9dKKzAGFINTJaB4o/fSz0C36/Fr189rqZeuwoVFttvoqE327LzFEM3Ud6xAgMU92T/4wjQqY+ru/+GAM=", + "page_age": null, + "title": "Releases · vercel/ai", + "type": "web_search_result", + "url": "https://github.com/vercel/ai/releases" + }, + { + "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJQcSzRdLlxpcbdoxRoMOu+o4ZkIjn6WpbBKIjAS1xrFrHDyPA3EmIrhkuk0f46MTAo4OQhbQn7vTNvFm6l9imrjWuDMdVvluSuLhBQqkRk6yWRRRnepqM+za/wU05/kz9c302G1sKfgAfnWN/oQtwKLTAxsomC2nKn67RqAxaTU7L1ZkvOl6/BGcO8ALZ6hJfpfFW2kOpq2BofCIQR/WekQJ7rwytaim7w5obLH7YLLDXG7jYYlh/9jslu5+l1L4+He14PpaW5VHjZNwntZLe32770ZjTA/x6yYUUWRdkVUd13SZKXlYt6xnLU9wZRHQPQtTK4R1WA078gEqzqf8rQg5WB02e2iKAHzjJ/TcxlS6QSC+LVI3y7pCVUFcfFIwNdzWRSxCIs+ndUSXfz/VCfOfrliOVWXhp9qh6vIZQlTCyJ7Yre3CNN3DLFugjckHixFQ/COKNESY/3vSomxipW/WHC24HU4g+/25pFKiJEQ2PFwdlvnt4EPP4d4icSD/xctx8YOrn2ZgbrvD1UHJVq44XrhwfKRt7tdnwGh1THc7RVfE0dLrrfIPFQHD5tWSN3Oixq5Uz6trMNtabXWmvhGmnZw9w0l1IZ7gEYcNuOL3jjL3QOHDsXKbqLTlxfbNi8guGYQwUwpMMGkVdTKskwoJ+ypVqriq+y6rvansnRK28Lw9iQo4s93CLyVlZT+1/KZfs/j0Nd4WMroEScAQsd3WofM2IA8HiZWWtxPbQs98P4NqGr8KLoIq8McXIDvIEfagtXxkOl5rrR+lc4j/ll8B141OI2CncUvtBQbhdfCinPkxBQZON4UldgwylksEXNffSub7Bb3i++NTwlJ8k1rfTMtgjSsl3aqJWEYRthQkdbOswF96E0md1BXLIXXThBrqeJmilUUIDmxZsiltL9bAIeAg/VK6n3apgTIQgbh3otX8gVqn3Ozj6wV220/S1ZPbl6vAp2+8puvjZmZ0v0q+ZNCMD0dzOz85TDpDB065mX1FzAxfGJP8lGCqZtDDj+a5Mg4dctWjLFILaErHHxlijfqj7kJFl6BETwV0L6cEVHdeVUi06PnlWn0aeYc2DLTCN9LPcuoK9bzFPktFKaZFX5+s9ZghRmyjoD7TwPifypNFH3tFnDr0rbrLv8zZIDrLW6zp1fTAfgbF47V7wWOL6quN/ckCkKx8cyrpXOuCl6V2J7hSLw0TC3K7U73FrV9Yfq0mHgfqjbY9byNr40FXnYYRtbE2SI0Wgn2vwOAMqrAh8qIBn5Z9OeQc2jlUKjGIGI7HS67CQ4fV43VmjqOYtj5WpoWNmubWKo4Lu4s1/grSA+1vLMz1JfVhoW99qqPNx7yAO2f1J/K+0P0mtwWgWtQQ2sGYvutJAeA6wXIvO4Ms2T3xlV+VPZVebmdZdqBFxGKH7JkIJxDSdyaDRNYeDviU0LEFC5das3AJYtuLuZ2CbJ5luCzlF7OxMjnnYHdWQBdwij854QEpohofESC1/SYnZszB0NqFIG78ZKwBn68L2Lp1PcqF2DTMxS/eRrxcTMFYO5XsaEVRw6kFqSqnEyUFHnfmIrDi7MLTvMqZZ4UYP0S1SwOU3FbrWVa4VAzhRC2TeHYhqCPo3mA4G4X2bnuzGmC15AzTsz+FRf8RyQeyFFlI9N07uEadfH2iI2/ZwSuWYcTl6BUQyvCzG27esJe8a2kWYGYVt2GY8xjHnJav2PUOckD/9ls93CyUQ9tqvYBb8YCvPHKz2STpV6t6Z4wy1WbxhdZcXHH/FLHzmBB5s1V7sR9Z8RrkC/R7CIO6a/wUO900Kk1J/hxd5GrK0nC3SL+AoD+0fnNldtQ65cYlI55kXYWl17IYbrc05tUKGDuqtDcUp188F5oA86qlvCBqEnK9JeQUMwIDPpNfVxxXcaTbqpqhgVbHYEB6YLgVt4oncX2tvVt6W33JLx+Z1Gq7iOnPLBfuhxaKkSleKoMx1Y9UWRKciL96KpfcxBFGHL1+8XNd/WiSt2e6+CLc/LvkzTcThbuz8l5XJGmAozO6EkXvJxEt23oNKylotFg1EY7cWFA1+GlYBAyE66+nkWpNCbb6FWnWzI4ltFgJEcELldjeN27moBmrw2DRmtEPN1RXpR3eQa19FuuUDM+Rvcr0ziYlJcCtLsom/76OJwxLk0USWdkXwsl+s9Y0z6HD84Gxwq49A0KsqjqvEq3qMcRnaUZOv6cv0w2zXBCJwdTjlwCpE9tuFkDcIeRP1JrAbtAUcVETkvGaXI3J2eoUDok+CN0xslOC1S4j+lxI5ZkFGaZvOrfX8y4mpDr+jKKWAGZ/HhBdg/6i9+spUdayS1dfMAbQK5xNXHsJILyh4lTIwELsl6BmrtOllqC9n63CgOL0xsXb6vshMN3Cu0d+bvugnTILZwUpJ1qTcrZ9LdnwCXwbEB7+1UYno8nHqfDTIoa6n7hWeQxifEE6Bp+uh3GCF9G71509NJbJBPvfwj1KfxGJ9BQtE64o/uKTsXidOijBRKZ0/uyqY8S5KPVohT2UIdadBdIF/CzcMTCSOveHxvqFOWTJreF3tZt4Y3zs4bSdsIt6x34fNxmC46XmQsjjqYDIX7fD0t2nsVbNM1MUj3iKSC6KsSH7JOCo09ckZveOtbgDwZKNTNBrie0UatwOX1PmyFyvMn8slAcNKhPkRpGf08nmfs7GUSdFAGDpKCt4mTMX+LXQ7orbIyAoBY8TPpPIrjCoeYjW5tpY4zW8WK/SYk4HNwsdsIrVBq6d8Q17P9m3vYfiFMpYeCpwt7QF73wVVXHCUpvgZKu0GtWbkUSv8g+iDEERKbTTM9uCA+SlG9QIHoPKqMSbleDsw/hKt+dr6erFgoihrAOlnMJnM6sZ/M2lwNEIkAG2TKmZe1ujuoxVgCxXj7vIoR2X7x3s+eqS4fxjPDniPP7orYUDctEi82f5rzksc8is+pP7MKjvzXM+tgjWIEyGmlDpETPL7+LcSi7SRSc6nBBpmMe+KypIbXWCG1fz3Y4Ca+Z5mNASEpEu4qPyeRz/Y9rspYV+4wemfVy6OWKfG0H2/VUEW6vGjrunrQ2JOw8pTQ47+dQ2ouplxv+h9PCT5YXXlCldK3EG4ijtX9McACLlHRdRsoLoSUNZZhhaP4wcOCbfTh3FrlLcqGEYvhW6hLIezcYdOjQNTvG9NjTruUySm2hOWo81Gpk4rULrr+VOLfGq8hckHWoieGgnA1Eg6gY5gNeR/3yVlNK2u4nuSgwXm1TwkTmWHIML/ppz+js/MGwI+8iEv2CQrrOSpD8ua5uCnlXRA98ao4znkqnaaOfjGVZ68SpPjhWFS0rm6kLqmq75R9EgnP2LNX3d7YljnsP7EugMpwITJHDTmA9EsxqspVORji+osT60Cnjz7X3HjvWh/nDsjwinwvyMEo9750CElFuslbVpDmzfeJsNPAwosAiQwf6rSAjxzjePi8wJEHWz8U1lwIAoDbOynJQ1950Nyp+ohyWL4ZpNf5G5i+L/26GmMSPQInSrmhTBl/lUXS8YFWNAaJshrW0i/4cIJY+EuEpN/41B7INuysyVfZIfZrk/Vqta+e3/KGYAfckUEkktWxFV5F5HTj34BgQZQ4ufwmDK1B+98T9164VjwFRFuQjw46uuRFCNpXOGxq9GIWCcYQVtzxS4aOLR5Y4SSGX08L/FH6Ixb+RoAv3M9KfSvbx2fPDXpvdfMsP1/kyUoboLaxW/q+A71uUy9WjaZZ+vdZFqqfZZKC1wd4n0D/yIlzA7DDbK+V2fY6GHtV+6tL9HgH/yuTn5ELgd7/YWu0m2k89aNgLsgaVP5Q9HItmUSCAUP/4797ejZyYqbJ9pJKhVLclYkZemz+YcqLqMh0rUmRpNg1jbn2iFJuJgMrkCtHJrcH4fthZEtJXx5yMuna0XEtrBM5je+lpXFFXI041uKotDL7TWxT0rO46F0mHOb7BRPB96fvDX+FyWmjtBET4aTCDmip125fwRyExI4zY2gPoRtvrf8rWgPSgh0uBbFxqIBYb78lVzMsZeoly9KvFkPJY6JcYqbSwPOcQzUa6aq/0QsldhRcc5RTW1o0SSBgX9it9coYcKNXbQTS9M1SOy+uGQ/8p6mfdY8KOFO7HHEgYLm9mnftOWrr4HbpRSoYS502DkPE+2YEfFXFGmsBbYvUaeVi9wrv1JBhtcWLWhSgouSjf9tknxUqfhqtk0w0hRhPINs9hgrLQCveiF00qgXjYfc1xIC/+8Sw2XIzSDLHTs69vXMswamqBUWNh5BhxeVFPxJCZ49enikfeiGlppiXZx72m4O9gvDPmIjU3aEZ9nISsmZHOtumn23oWFesDeUrrI3GJQnzykqPLPVvFOQejHAR7tvd94DRllCQYxFz5q8bGUGEsV8a30uUmGAM=", + "page_age": "2 days ago", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + }, + { + "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDN7ggpk1Bol0KuQEQRoMtSYSTWqpxJjaNnh0IjCV9CzSm/tXF0i+JKVdjFvs0lGTHHaDo233ef8+oHDqflzyiUAIBrBkozCvx3FIlHwq0x5TAEp0o5L/u+96ntWm4FApcTu9gGX9CxxLV+QYSe3P4HrDpynlhamCZP+VeYsMkk+ZZuT/OMF8I00/idY2HpAxZIU4FXhnUeoysXNh+zur2gA+FXR7JiOjUpOLMvA1YWjx0QpaeGNWnFrZ4cRC1yd28/L1m0MR/L3EDl1eGbV5IE2wAOedphYvDVIcUs3Gl+RIm+Cjy2DTvHkyDgZcW0fYgVOF0NSomDcoR7JPsMcq/UTTJ9pw854XSbEiZLi7Rzm4yuQuBbJrGyNqw068hiqNDOQ6hyb2SkMSKViIRkzQoj960QjhFl56aQ7GKP1hzmrHoTtMVqArw4MFbKRuNoaGmkBoQrzc/DXUTCkz0wmTMEbnF97HleuxOsWpjmH3tziVFMhZx1FxAb7npPQKqMxegNpGW2BYDqWIiUxIGL8yvbmPSVveKQldkcp+PUpXrqNWOvAhbKwdvaiP/78nI/zMNaQ0TMm535JdGB22MbZGtW39GwapEpfGFVoSElv337AazTVsVmjqXkBm7AjEfSA84UsS5Z+b3oMUpUk9WiJTg9ayS49CC/8Isgr7LHU2cB0AoB4hGtOg6X7k7+g4/ZD8CUCbzWnDvVefac8aU3EIC2ohKbkDlK2sP6fAmL6REgOL68u79BOCVYuiN4AtZs2t9scKsVYPT7keLYxVuZ3y9TlhaQ78esOEM7gXhUubqEG6sp4HXluUgc/IWrLxJYOifERrl7TulUUkIAGPG2ic06bEpsuzI8UlEfvdZuZE9qFT2i63Erz/UmM+hBaojWLXrklJ3o+ZfOJPWNbyeZhBW+4C0Iuk8XtuBpTNOlG3dmvNiL5+XlGBAJYLviQjGvOEucaJxLjlF/3PfrKG/RigUVlZABV48PotUj0g2W4LeivwRIEMbCobmMd7Tfprge/PqX9D1rlNqBST3NUJZ1nDOms6KLziMOf4/HX8GkUAffBUsSRVTepYjiAStPUAhrfQNevvQdm7N0iVm0cJ3c+BwuGI9x7Xdk+LgBjfATnMjliq5JzTkRqKnOuTi8LAHdRojbDvfBAt6MMViyaSIRPAEwhmMnjW9zJMdJgTO87PzBEL04vl3M7terqXYjMo5x66Ujxux84X9X4k9xKZ2qgazR1AlkC5jNU1125A2mv9Y2pWb1sHuhiMESwri3+SAN5AWeBd9Zf0EswHpg+Da5Yzqw+R4NSvK7Z5JPKFPzRbCqLTCq6KGiNP4nAsZMuHrxhtuWRjgbO/7HVTXeg2wHWBXtaIzqSlkkLsOySnMtYXFF38HELLxWzDD67LMU00CMp8Hif/T7RDg734yYU/Ogl+kDMwR4SfuBbW+a4JQdFgmIb1Sup0Os7hQOjKGeALjIVxQ/w9Dev7mCU+rl2uxrmHjvBIlqP34iv9/JWPVzaQcL0f29pOP2xXtVHxiN5vVcrD9UpUvtTZmKvPI5K+dZhqmb60IPVfg9mdce39eSZO+is4LzWpmcEDvsuxeT2jnbzREmSUZVDpkukMcs/A8IBFaktboHUDlDiAf3K2Kmp6mCXn02BXFH5eNK3girnMF29qOnKdTlIE5fXUw5CLvObGDoZ0WNvHbTm/INmD+abWBfr6MWuyFm2kvfCnWrOH36ErZJAre84aIF4tO9MVp8KqPUHeXCyEYyGeANbH7vbo0h7pe4uLWtCGn4F2My2vWMgekrPEb1ItIQdrABVVYfBG4dd7y4k88G+RCD+xRFvBzWRbWMK/rMvHWDBO+4fZilFgGmJaYeJYDjRXfzAmQ5sPnYbTt7ikjAVD2mOMDdEra5By9w1E9Sc9/BobK7UtRfuTFbFzhtAFiyjrHKGwkr7H4QqU3nQFAdYvlacdZtVSjo32UOdajY2OJyrjWjxkJUtkWeSvGIDWC/HQUoDI8w9R0LyvBnzSbJJJ1eGDendN0QKoyAjN1Q89lWSsjbkez3CAtFCbmU1jQGwHcfCdm+60QeNubytmAXO8xXa7o+WMp9iglwuZzwd+fK8fOEcuPnPBPhWQbEkrJzKO2yyvr0WvqWpraSUK71HlaaBFsaqF8V6Bq9HBC+lz7ZkbQrtIP4dFUg+EBbRCxxmmAQ61S89ucsRjjbYgh9Gnilw+HEZusdX1EwXMm0eMhQPsew22TzjEFI+jByHs0Hw+g8157yk6hDBNJCHFbAObOIul+VEPoUVuMhPrdw2ZrLc4TPXVS0L59BxTmBLvznP2GeG+H7ZQlAxZzsOWcAJT6FlDvT7xvLdl5zrbz7nKQSj08zrhGgtD0mtYUJrmgcqR3J9B1yc74rbbfvVfUB/rviaIl5WjwSzTxCycVtQRpa3CMjzcccByvHVdCpMHoqGQ67kmXpdnD7Qr0V9jAebsiSzZUtZjNcdSNb4jxezow/4b4ed5e6Qk6m8Hy9B1RiuOQTkUK/bM8nY1f09YbaAXO6zDVJWvS4V3NtzcDG1DZXjXQMr1SQS/4AxQg+7HcC4GBAjBPC0HEkis7uaCv1pVtUIZMRc1hgIsZXkjOOyxa4BGCKB+fncpx4oV8kN2dTJ37ggoiH8rAqxsOw8oFQKSSNFbQr7WbobfVnvySnFQXXF5LAnmb1oDFBEf/vzBI+f+uH6kz883UIs7Bz8xl5XqkfGehS/6gLPhdUVkflRK2jVQWZA7uqfJzsjXhNBFGMz0PHSg2nJBttXzVkS3SmfDaXHYbO7gMIHldKqcUT08Y5r7VvUBSVdjYOwUeZf8AA3L9amC7P1yq7LGqswTAi4H+F6cCg8JV0+nQeiSU7G7N2eJviptfxgfcHHJArlh09EbNESrL0izKHt2hG5jPOvD6BCY2UXOdFI3+8bbedoraT2xC9//9JwZQ2dDUxokCoD6P45npfVj1llR5AJrdEtkfPOG91qO1Qfcz6nTN25l2qBJILoyRZxFtuMlE1hcOdLYjGhgyl4TdTwcamoEtfNfPE0H+2tzHYtRrfrSvKjGC4mWmX4+RS3yGXHlfAvpgd9Dk0TINcgRe3uCnxG7Wjz1c1AB4QjxhCSeoqmHL5uWRI6/oxPhYy9oJ0KaragFdfK5MDaiLIex3yGE135zFao7X0C0cCgItyhLcgZWVWc7zyL4Td2U8vitZC1aOIRFPdjAkxDYMivYy/C3OdfYzKy6hxQfRQhWIRnBxZ2vvNoXF7o/2QGHeKdFKw9wCJ+ZMjldIAtEgFT9yQy2SzfKTCDrVKAs1e9LVHHhVUBSPVGZWBNCQaIDDO4eo3+PhQ+ehlFOE1SDhaBVXO/XDaEMnICOr+6y3rp9YIY81G+u5UDRcPK2Vqq0ltKEYeg9oEwBdOsnVMzJC5CWdNi+7EmZIT83RdK7CTfigbD2mjJPGGPCutwh3nLzL3yZmhIuESYLgbj2MJAW5Uk4DdlE35N3bQqPHoYZKnRcZTEg1cOn1rl2B42TMoO6hoqDLTeF1IMR55UnNypqoa9OagswWRp4QRPftT1mV23+tXn3Za9UAa9EfkWMV/yWwarS9v0n0d8BwIf06FsBKclgMNVyVmUt5ZTW3S8HLH3KKJRJiX7+VsVAKd8PvbX0F+NMwzofKRFZl533PXianLPpogzaxd5jf/BaF7LX2O17WGKcZBe3zPnS/75TGMvIsmFOn1R5O35Jitgqv/d8rrWnmd2QcBWGixVRYShMThm+iHDC8tnGSEmf/t9Chc05eIX0NJQrVqLyIzCrYabdAFTBSszq4w+GaYgJm52chFDc5H7sybGHWbIci+ktocsh2msmKLidncSOSuALV+ZgCSXn0w7s0Fdcmz2bSMCNVHhMaZCiENToRM+W+V2UkrR0+YzCQ3qFNc7NrsSVAjWffmQVuRgpXUR/oIDgQTRFbC3fNzP4tS2I3sBu3HYFUQH9SFLAdU/+EXR0hJfmUU9u/ZtMQzX0jdDGID8IfU1DKYfSwQu4a6QHpSNhsPi0ipHeESUSswAL+XgShkG3qyySPF25bX8eJgfdmxrasrVeL3h1vHORt8Ljd+dNXTMbMET+l8GZge5B+s1D6iezLw+I5rgjWbQu9a21BObkQeNnBDOPF0QxUD4zYcx49N3gWeCZwc3xJKUMRduDyv41f4/7wqSRwTQpvpKTG1NhQeu/da+h6pcR56n+SDLuv119EuATA7d2dVaGY6jueJ/iB9brEog5CSntbJ94Jr6sRN9J8kOe86BmMwgkN2ESSMx9WKFYx1+jes9R4/HiYmCv/2StyNqEaIM0XBfceUrreAuJSigMe/aeyMejLz8a/9KDJn3ISivsJphhVwRyC3otlZ4JO62w5+3hNvuJXiyR6Juv28knTuAVxj1hqPaYtHYTr42ERcHjzTiS9mxmm166Dm+/qHQd4TaJfrLSWKCK8kTluZiec5VDaTKCg2qvRkx70JQKSJdPgcQrtrJcOey1GflIQXe7A4OelAC1Ma3HJwHkbVn2hmpq4Kp6reH09kvr6hsJP0AMEYva9p4xBXsJajPB72slq74n4eaTNgyWq9paScWgAGnTZLcZJSTXTJdBLSFA+YleXlcKAfX/MyHbPVOfcmmGeEXNL6rG4L9/BlW/Gf/EthVHcVcnOF9QhNaruaaxkfkD7Ri2DOf8wzAs9J5B7YRreNucQt+XuqNJOVKbDPfodxXyBFFRtb8CK02f4ofyy9eU21jQWPpf4/d1d+mFLxIdf7rl+23csCpkwYCGZWdyW9XVikMteWJuojavBZwxcXdm4CzWla/LRQhP1PhK9M3RvtBpdvNS4Og7B5FwLAVEZujtdBijmwtt955WPkz5U9D344UiiNWf1taADVexcazWwUNW/ZhQH02d44Xm6grRTCQITOAA3Pn+kRQPwcP27jPSLC7r+QM2mhw7CIXLAzAxet0kIjb9+hto1YUyEoloUehv9hq/Nt1eznHSF3SjbdA6vvBCbjRdDDjG81/3VCt7JVobls+SU+Etf6ChRkTZUhdP4yPg0SSyQDNOqxZMQP7OxAEu0qFo7T3il247WY9ZkN3eN+gEh59kH5+tllHzl0N9yqvoyD+sJm69xabdQ+l00T6E+j/7Vqy9TseMKDsf9j25tHbXKCPxc0DyhhRW0wTr9+IWXbG/ZLXiO/sJPuKi6zETt8xVnnEvctC0DZujSGLMapAtwyA/WB9JbQ5Q5uI582CEUxOk5gu6FINjEALyHMrpO+hrcVVLVWXTnqPZwTJNfUNlOfp3y5WbSX9p08rSDMjl//nDog1O/HXNeQQtNxat5IX7OAQevbMGe2dcd1O+zhgD", + "page_age": null, + "title": "Release Notes | SAP Cloud SDK for AI", + "type": "web_search_result", + "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + }, + { + "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGnm/N74O39+A+ErmBoMSivsWKAdEnTWwwvYIjCHslw24d/EHxd90S2xkn7YKcnHQc9ApsVsQONwuTDFF/ri6zoZZkwaDe7LyiJuYzIqyhQ7Z3t6Gzp/Msg3z/gNP84RnSe0dB8CHSfuWUCOYQBvBrLj8CahOgfT5c2ivpcENOwCarLJFTGaBzDV52PyOZGxavtrdCAYi6TOuj/pAf+xK4A9fbj9aQIIk55TpXqHhGqOU0lkmkYztrT1SFD+/XoqRmm4iVQTMFborr+nV5soPS19kxU5yog6P9VEiD9bbXL8iaWrVXLY5qo3vx0Qq18LFTTjV+ivytO5aEKpdP3eAEmEQiCF9IucKl/NALzuogDy0CzdRxnBVJy1N0CGUUjUbl8QtRi5dFtgPQ1cp8gD5vI6XtMIU8wBdSrnihnGjqGGf43VOfrZ+kFNrL0JrH0BGibXM9CQy2CgQCmsuT1lX6FJAKNolY05yaxXfXbeIk5oF/sqh/JDNtUIapZekP3Q87fw8tIP8PkHkFaR44EPkKFU0q8rh7YDW09y9mBqZ/1zNv9+3n0fnD7im+3IQSZFiKKA13kMJ7cPf1aYkNQYLdyrKELyuZb3rxvuh+a0NqOzAJEzZb4nYQmHWP/QGfVuTsE5WUOfdfvBVBKduZ6xOiVgVC2pKm2cfImCLeBRAOlvXyypWn5/9BCgUIjUeV/kmz4CeW35G+qRILmABS7o95vJgbFiMSBktFirAMfCjuSg865kP0CEshPZotoihio/lTxIyOcEbegJtI5Z5fHNVZQi9JDJq26+MiqhYTA0AxLAeuziJ+6ypWrbKQhyuRyT4DeZ5KBSM6szoxSReXuGJMCmlxDiiJzspKrpnzfTcPffZ18NZaEuldtmKACQzorhMKYw35cVnx0QvhryAeHcjr6KwFBghs8qvkvu8Z8Jo5v2vHLPokw1LLiVq3M4mwgeRgvBDiyvzb7GegMM4UVdsGtnqLj9tk3Cj4MphdFhFILjI+PqLG+dIKsJth4yeWQ0wYpXX7fgMhrqMs+EPVAML12/V1GMlbojECYml8KnQS9vvHLWa1FY/e3/HO/gCFEF+t3AV0Dl84dOQ0bB3K2qK5SKSUwMJg365FcZpMsJC+tM6tU8DAoXBKe6U89WcMbb7to884Sg0pT88gGCNq3F+4S3JOd34VoEh7mL7RzLS0YZBKAJB1fnH8NZzxVGSagOa1XHJ0H0b3P45D3JwvA/mD95lSlRRSHlqgZsALGvIYB9M7QgnopZb6PL97nCm1YQw+RwFe50g17qhRMmbiqIsPoNFGNvUCbnQGW2PsxAmIte3NQzVB/L0cLYIcD9m8lQixWyubAWrwikLzyN3whyWscGYh9NXLnWpdfRxz/LbyrbEEAEge37wIXNKw4PKjVgSAkTRaU6aJwvqjx/Xdzz8HFdHzpqYDQ+L2Lb6ndv+Y9dT8GkNHOYKT4pmeBbVrGc9NFZrvv1yeUKqAcsJZj1m6NjuZIdsUFA4D2U3X+WjkkK1OlGXZKlXLvDfM7WBoWowiEzqir+deqh8zxQ9pyY5oVLWxJdhYJYKH9RLOzJWqMx2e55URbpFi2I0s6xvdJqSsk0npFHUWbW4Q+ztHYKHV2q6Cm0DHXt0M6ewOI+vJnykBwuh7ihXh0p6ML27i1QhGvQNhO421oSWQGHBxKPDfZw9wnmNxbRTU6flARbjKJFBdxh63+1AnBaEmu/uoLEyfmb2gK1wfcwgmG/FYDROWtkhcy8xGCiQROpezuFN9b4DFnXATlhWTZO95Famf8jb5oIADLH/Avu0HEa9Ma6TQHoKPiOTejrdd2NutfMayBZak/pRI/bT+1NFIoyQrEVvNNrCsgMpMY4g0l72hagkFhYaMvLC0IGLa1npCwJBAOvXGwGrXTLhxgVWqfEBdt8dv7WVCmUMwOsF5ZpS5i2tPF5FMPwJv7Iq5r7jaDaMwCEsakaWigmulk+mxjE++RUKwpGh+amYEo2ire4+r1ItQ/Xjs/xjMEQFf+NXKm4TjbxAi8CJHvXOtQhIb8mqw7FVz2TEZFSITOWktv5EmsqWnzzyVORuRj3ElyLCKc2t585inxfMpxlAHDPDE4KuYjJTqTai9uRDWMxillK0wD/ZatqEFTQIUbiyE9RKICjs/PuQnjoGahrTZX98Q50kEg+UTIw3r40VFohBfYiZ0ccHn4sbG2gBu4MNlZLvYVuWhZhxOy9I6W59hzX5eh858LGG/BP0Eg9gPTBYxCxtrioa26QbdEMRfquNB+kyIwIHbyaOV9zqfMp/fG9pASBe1b3zgmOiaQFIKr3gjzPYk581O3k+e302qNVpCV+MC5tPxMopf4GXkCHCqq8tJC6TflxAnlKJUfg2xkIhRBVhuKEVpR2NSrWoUsO0EiiN4fQ/8v+OfUlJAOLJjszyTUCzdOS08Ar5dkrFL2U1AsJYbjpCoj4nGYstrm2gdi5ybMzzQapay6krcHEP61a2vztc6E/UjiEqOgjiEM+jV+62e+jOPTodjBe5scC+4hRqbFlo6PS3+h4YPAK7vQMnJKMyYGNhN4jmzrshRXAkhAlvxHfLyYwwDlvWRgtZyV7yx72FJkvAN20FHtlcPjpu4/5Z3nMxIVRefCq14Tm8zSRwlOhDJt09qHMT8i03Qiapnsbmz2LiLMZGw43iiMKRIQMHWXYUl9l3Pz15yO2Oht26cMoqi2Pkjr93S3SiMzIuTV4haXZwqY3aPhhLLLW3wNUR4FwZbEzwvRi2sqHjo799q54Ds9kT9Dp0d7TNNVLoGIaR5o2APKaw7KOQAI6spHDRIMOouHAgzdnB4Jqdm0sZSQCPvcJrziTgyoTWaqPwbsq6VmDQB6YTXxflM7pT8iJCZBQc1WGk0XDC83UyWe7ix07346hf75Q9ujr/dSO7k5vRQ25oKKaWGB817drvbaX2APA4NFHMSgAhHxhBV28gxUeJA+dJsjgIhaI2NUxU7EHXbNBCM2loQ1oifNArc2JXm9qYeRH8eA0XsJ5JMACnFrK8VrUc1eY8930Q8nXKSs9UZ0O+7XhauXlUKOYxrhJXw/tODGt+VP+JrUI57eMOnFuhflFpUUDhW+CnvfeKt/m8wx3F8grOGjlESbNKFd5XcCKnwJAy/RtJGBijRXWA8mMGSodI0desBflLiO1xzRnL1fiyfuxbQ2sMMwZUor45kCd8zxe6NesK4cDgcLpynudXGcMwsjmc6OaUHAdbWx0nKJ6BdOvJ8V19dlyMdpphpmzsKF+CapxedCMygmqg/Y0cWHmVlPDDwYusSk1wHqNVMTFV9HVgd6IioTZvFepExX+sKO9/ZhHCRVvRNE/xXlzNrEgrZBaNVKt7uOkz64Kc+EMvPGIBE6lN1UxNyFMj33MIa3OVFpEoXgythtcRDBURb0f5FQrJtOiMMDwmDbYJpIeyFpEnkXFXcH8hsfWhuFgiUpuKbkMgTonn+e3IOyEnUkQoVWVAs75zSRF2+hadmI9swp4cUvcn16PuWo3JlKujVWlMneb1rRwBV/S9BiM+MneQqr+vK72UNv+EJ89c9v5moPmyDsGAqR/H3uDXMqZVtUhQIpbM8t8eaEKTHmtC5XairSXze4WVuAYAw==", + "page_age": "1 month ago", + "title": "The next evolution of the Agents SDK | OpenAI", + "type": "web_search_result", + "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + }, + { + "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDEwcBIM8v+I+UvRo2RoMJ5MCBFs8z0DuLxQ7IjBiQWOrFiwM0EkQ/t6s+ESWxJ+iR8WWl0z60OA2LfGjHRjckq1/jikTkJUVQ4YTGpMq6x21x42bX787bSmYAv8aZPHb63CDU+qtOPlCQt+di0PykcWdhCYVjpRHiI1WzAgz2ztKPCzLGE2tVHxqgVBwSVPGtS3uHblHH+m7oMXwPbx3u4NGUYIXc6wf8gxwtXAYt3Bc/13GeEmLd2hc30U/coSuwcsRLVwKoNiKnwe0uXHOCbckUGPxOPivfcvRzwOegx8Gx3I5sbCuoWQLUv5BHydQ7H4FF02gHQ0ca1MG4We21LBx8BGhGWgTve1DTxrvlTvB78hmGDBUeoBCAlYdjewWQBFFpFbl+kGGVJB25aqsABzb9hlgdUddmPjRfIh11X2i5/sA4CkKA3mzJFWSFqbfstc1z718Gmg7pMBscMKQ1s82pE0ll+onako6PuHgqaIzqpGmh0fZpFoEZqtud0o3HVnghVohjenvLlIviKjvXGXAXOis0dhI2Kw1IehXdicp7Cd5dJyVCGvjuAdG9ECd6MyOtzKu/srQlF1+OXrtuoCRpOevgjdDwCwZopAXoQUYJWNdJQNseB96KPchWA7lDz5Zueym4pSiiFpnXHw3CsK2aAiSMeIsZEqcKEnDbkSfXKDtpCqs34xNoJvgULnWHm/DbPknl+67TV1v13PCp3sJOHjnen4iNsDyg66RLkqvQ4Ufsb39k8M6kxIBa7ypmsyXUlx2VOnFs8P7O5c2XMR64j+g2fvw/Cp1G9EfeZgQIkkVvC7h/UeALGsmke2Oa3dGLDQHh8NKc6OmW82TXU0kLcmwnLJf4IiidCtckmsVeLt0DpGTb8ItjJnz2QmRoxTKEFTYLz+TphG2LHrIJSsJz+ZmK3Dg2ZQS4cPTD2jEs5EhGghglrntgDX+2woOFUqGNnQl9rIS7ClcTToWZlybw7YoRpq+3ZMeA0AWhI7irZoEN906IcxqalSu0y7fEQkDAHtwqmy0TkUUIOpGzFPsSYsQty6Rmxv37CFC9iEjzOQ24QTvRj/sjN50VO3y/NkbAFv/ZvaVrpkmI0VibOjr8wrDBrJhCqgeMMVa0xTmNF3rpOLz9Ehfh4MOIwmpx5x1qq1VwPZ4IpMb7ND0OH+IxsrFdq9PTXZW4u8muS44Ifc6wmTSUsUIlGS3kDNvT6NkbKOdqD6XuqNuYDiVTr2kkRto7H/+oCfdbNSN/IkbY6nR3xJsTuP46etZumDg13IZaV1YUnh1ZWlpj4suAhGgUq7tiSa3WFunm2zuCPffK8WxE8ls/eqi+dRrQeWZM1CgYHXbOPJeAExi+U0W88yvmMz2YMJDAcTyaFANO3elyvrzl88hr3oVqezW8J80a4TnQJ7ubfgCUdNu5BrIZvThhwvH4auIWFAFxulOiCp1hwfvI+tm5CG4YArnqeKE0DnP9Q5hUNwXLpMXAIGx6rrM8ttPXTLFKp3VvGVevB/mGf3PXJgzcCeFpwvaPCxBc1KqJ5BByuVWf6lMax/zpbFJATdomLibDGRkpyNgnEjCvLDfEt76fF8jJjB2M2nGYyKz3BkQWVWFl450S//crAUP6th6O0uadcLNe0NIA7gTF1CB0E4r164Hv4dzaX5yQMkSBqIX0V5WRfdAQ7nrZg5TUSq3wiS/Q1IGSEknzSTgvdjLXhgHIpN2MmeDb1XS/Woiag6YFHhaZstYo3bGW/g5lWpS2CbsLHHoMXRd1ZmxjVizAdPcXU+VbdUVoWa58POI8rfvEqLNFph1Ye8mcCb5TSXTNJlwkagl0UtB08E/D6gZLlpz+IDvMsPl51OIcbcsDMIqAjWI8egmlmzqtXsy53QGSj4x2SJAlgUM4hCfy+4vdWCatiotuVTm9zBXOFNo45pK3pPo4vkltUdNb/TADC9JyfFnQqMHhAZxnac+LacefxiiWym02PpgakB6NYVnNkVZ6ys8rBY8xJ0UcnOZJ/CgkWJgdt2pOy0zMVdx2gO212S8j29YpXwgVTrDIFeZ8Nww3qSrn6ZnJJOtSdYVsiD460mZGmN447hx8sR0B/45TR+/UB8Mq11ZCnsyWEUZKjmdIfd01aL9qxVv7AQieS7B/8TqAyTyw4khNlR6uVfpO1kGBW+z3x+ESy1BiwHVmxY+DuufZH1Ia7LDjQnt8ulTaqdjRMIxO1yFf0gUBBhtJNMsYjWA/HT4yxnXtr3FSHuOtqx9NDpmAMAeQkeGptK8ZbXTgUfHkHPQGHv3T/d46HrVYSREviOpXEjsRdMTbJ79u33KJSbN2/F5wk3nwqHYqz9JTqzQ+dvnQ7zKpEkaI+z4lMid7VyUEfj462u9QcvUAMBN98Y/1QI13J0tQ43nPrWJS92f/gjp/2H1bt7CJwUp6OsvZleunGW59NgQ3qIZRXjAVrNDnb5AMMQ8o6sIILpNwgcKOSf1UuklTXS7Z6R28q6lM+CRe7Hh5mYgcyg+0Zzow6psF4yFUdf1FkD5dsNXdQKpST6hRjE3nlHvJKPDukl89r7I5JapxkAx8kkOFNJ50El+gGTtHAPyuEahIqcdZ6x8TmoC3v7ORdKkxgOK/MxCq3FTxzRnEkCXhOiWkoyZBL/jHxkcYJJyN7oTudoeCmf3+w1QIvPbzCn+TDSgqxuh5hbwXrf9WELgBesId9diu0iWssBU10o5MCSs5+n9iqakzbB51oLi6lRlDy8ERroGEIvdP6R32XRRX1boNMCaSAfi+y3jKky3UIY+dN9kSEIVclvm1ZHUTYK41tbjZ5WbrZ3W6Uwu4eCSBBEwM3h3dx9THmymctMCjMRkkaNEso+r4eU21SzVPiQvTGh1mkmBEou2Q+r6cWveQHnza14/LTx5+lQbgcZ3o7qMI3lM8pljk5ZD1HtHBEKW7nplBUedhmsJ0u7zvd7YmGu5FdIejjKnS8rjyuEwZgsAFneyVXWlXguFzP9GDCHzWcZlyFRlze0+4lvUgkd/SZpZGJ+018HFIhiL4q4OntG2mT/f207XU8+TyKE13YWUv9z34CKKw1u3RTUD96QAFQw1z9j7fQJzm17zjTNimij7xjulPFLvzirOoA4DpB0G4rmYIskD4zLRRRs4QvSN2ps0rLChcrbiVpjnPjKZCC5DRuNKACcp77rQ2R2uL5WaKaEc/6CGNEStxNSE/8J539yQKXKoaIeHEK2Hp8pQK/eQqaETpEhCp1/Ov5DT+Z8cSFdiUF4x2CWP00Ylkaqr++l9OZxjjHpVcNxhIYY1zuIVuQpqbG1NnEq8xOWC0wLNn7HAfs9w/PvtSKzjKPFhb+rlJUkvJCCqN73+9qmWMFKdm8cOKJ/qGwF5/m8Jx35tfRTSD6aQ0mfj19mo80DdD8DHWeU7uP7R28Wypa61X3XzdHTUjoRxqNV9G5e7vbvVd+AE2/3pQTwGnit2OB4pMbMuOpa5teT8JfwefPEjUmaT7JneAFjCszPkaSyd47cDbhd+icWRqZu/U1zBJz3IjnT6SHetJSSVW+c/aTLmjjqIrc33OG2zfTvvpsZK18ipjoIX641OMP9HG8BGJHr9n7HWWt0akEJ+Js0Bz+vzukRTP5DSRQ7+XxLi7kaLMJ7hZN6gDeVgN0KMpQ3eMlkuf5S8OQCuHUzIyBX7yCYwonxUqnrSn1Do7bl6QgHiCI05p1cpKi/JwP59jvtJtrfD/8zvYaDBjgnUnlz3IwHaSqSSIN+wbx2FN/64U1BsUsrkMXsefluxcXJlHz/be5bb7/pDRl/DBGm6WEMTqN1Uh5Ma1I8jPXSd2lQq677uDi5Ewv6g/R22GuScyFFX1nk0CFSDUG4TVwEyfhiu1ZY4/2CysYFVO0PCvb85sfFga5fyQwEXpCwytxRYAGxKn7UOWQtCnxoMELb/jM+qPuPyMWR0oBekjllVCEugfisE0J4KYXyjS0Nu1RjET9YrHn+77Yvpn2zPHt6X4XMZy/KOI/x9gObT9JOX6Oq23gNHwJY+XGVj5hAuiTiqN7DWx+mJstzoEpsFCxUQF36wwwmuNp8Emhf3jA7MElmGd97kVwH7+/xpIpsQEOXZpoz4oezVS+orgAwBwFKSdVpafGGgz7LeyR8DTaWE9UgEP8FxXCejeNeMjX4cQlSc1KZX9iN4mf1zgIdagc3qFmYdLXDgsEYyA2nzxDB457SWjQBKKAzu8rQIq3rgsVeid449LHKTUVab+Bwp2/CczMF44WPhHXqCw+WDj6pDOZWlijAIFq58KGQLUVgzUSJyfFrydpiim859tABmPWIOuVRqPRzjEgMwZOOlTtjvljeDQFyR5GV1d5Zmi/lrfKshoXiSOoCZsDJ6VjmSxBGHPXCGkshJESzCMajhZGLF7owyHlnpWeuBgA8iO+/lq5bPnrSbLLC173BD+E8DxdJvAFLehfPEwfTKy4+n8QqId9acz3I8kL7ozN0P7n5tOJsrCPEkBUCuXosurldTnz93PEo69MGpQWUTQxKGpq5ngxbCuJuRALmwwqaY4ewzsQF4QiOhsnZfSAseqvEDa0E2NytIY9/Ak46RJ0uSyIV49dh40Rk09nmRbp1OlAeVEZ1SfSc5kn52p99qJi4SVtUgVu6xvJKc3Qg3JjwXLlWsUUN9eZ/79ewa3uyYeibujCxdD1Pnvf8/AyooBmvd4Mb/Uowb6SbbP4WqU7M005zc3o+261mqynH74sEsEjnh36+pCPvzfmv8zDEbRed8yKVJTMTGrpIf/lx+eZVA9vOn4SWCXmyy/g+ClM37ZLXbhTOcAvgUr/8NdZnyHvbq4IaCexdTlj7Vwn+7lz0r2H7KLB0JDPWd20BA6O0ikIVs+q4scq+Icoq/vlyT1zJpL7YmrNea84BFZ7qEFS8+XGL2DzqRU3ZvCIDwp2cMTyHRwo03OcCAm3wH7bpRQ+R7qUxP/uGblj9himXPJIxiv45+88QXYCPhl84oIIF0n2WX4ApKtNK9jXgLjq37r5VPl6N7CoK30etGYrHgW0NzvzvrlkELmgLE0/nfpYrFc28RoNRbz92JIC2Y+NiirCNIG0UoaKaSbn5U6p3yLlOabb6NZxZtz0aH2J4O9LBrBuIahI0oc0VDkA+0l4NN9Sk0jLosh4t8amNA+m8Ji029Y2z8l6YbF7O7rHb1xRnT/5JUJsG/iNRhzj0B60tiCgMf6/sYAw==", + "page_age": "April 8, 2026", + "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + "type": "web_search_result", + "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + }, + { + "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCcL/yWoxQYgZdd9WhoMpJWc2WxIyUJ2YM8xIjBH254X083nJiyC3C27Zojh6fQNspI5JE1DicaOsQNqmvDS9NLd4WaKzpIhJhX1YnEqhxHHXuh4QBX3gTONGE/U+AiVuImvhHojaqu68yyvU/x+8DExAKuirhJvaL1TfWQBM6VpLk7xYHvBkk2QJyH2m3zE2shkqf5vYYUgRRrs6CYHWbUhVEcF901e6zAOUqKVaNhFdJgsbUFNO5WIxnv7Qwy52r49uDaQLx6GwOUlBT2QKmW6yKMfTPPEwjNN4ee7OdyRuGgIpdPKplzf8USJzU4R7NU9GEiRlI1B6KWQVQF2OY+Ep0mhqqg4m20SvWwkrDCVVe5XsVGyJMewJR4uf2M6I+SH2S+zUf+GjfiZGbQuUz1Ae9FK/d6I5/VDBWy67eYg38on3h8mrI33u0XoYdYCu/7gJl8M8VGB14IoNmp0VJXRunsv3iejsKA1K4SKkNjQxom8XBIalh4Jv6tVVjmJNmaDFkqMmzI2/nwXuUT1S3idYEvTkgmWvQ8G28fer45zgbkddKW2qycKo15KT3kuZqaVcp3rbzSIyU2iblGqXHUm7imF+MbILuHH2ITEvKKAHnC3ZDp7JBA+oRtyZImNRmTO1tFF6q3BrB3xCoC8Ozm8wATQYFZS5TSOdlWgcDNshAaOlpzv1eAqlJ38OVIu3Y53Z6owfMV31I8Mbsv/YPSxlxTVN3uisgI0JXkXcs51VZfUe1NbMfjClEwF4ARqI45W46Fyl0i6MZRcB9628Yc+Y6pO8llKM4t9n4Lth3pUj9nVC+T0p0dZ0TT4FQvXVLok4sY9PkZpqoZhvPTZKdwqhlWJmmQFM36t1J2m6t6GplBFYyp/K37KWYrKC/L+ZmCWO5wkFMFOJm2Nct179670hrkXW2PwMFYJQI3IEmQQjE+KGPEzLFawNJJkn4GSu0jFAYJIwO+8Vn8EfKPCMZ3vDjxBFsbVVjEzC6qQcMzRnezvpad4pj4dYMHIQZj/Di+s3MRJrYE9uuOKP6JYMoJTZBe6LHu48MtRA4KPILDMyJ+m7g3oY3APVy3tmV5hv1KFnpGF9PYC0IfIbFs1m7YDoaiDA+7gk5BLpw+qffDmE1Dv4cpVQAKEsj7Id43/v3Q+ulutrfcw3lTmf7/kGNBiqgdzntdOWg/X13oPb3V/5DoAYSCNjrNXWba6p64Kyvy7ZgifO6LRfHBR6CwdLhhx6wE9hr77BJ82Clprg5gXIRmWKZUpMnmydbrodrLcB+Fl57LoElX1+4b7r+Yg3EtG9pBV+7GJeuc9FijUwHwJ3e0+oZ2I7kCwNuexcgdd8O3YM7I3ox74CKKlVd0oklK8jzqvjzY38rX8f1yM8HTxC/8Pl9Vc8J9udlbPU4+Jsl5uxc3aDV0aZCz6DM5XbY2anqI4cRThGFxpIv3y/WLTDqGINhozAB5LlaR2IY5bw1ioSkMBrE7HSNdwfDgCSe2Z5ifQo/89OZiy4DKLdZyKGGXLCAeaD/xBJKGDB7wAu+FLBDujTcPh5KQxQ2E6i3Q+Mjs9CfQH6jwXfux/uEIPc+Sl37GrwXNG4+wbEAXEx7r1JkTisRujkyFKaCFK1BTXvNZvaxAzyVti+CSNNhZ2UPgzWwAjU6vsa5mnmKCQYQA5CE0FltMiJ3vKq/T6mX2mTPLseD32hWmOj7Pvsn1qnaovh8nmtEluM2U6PSpJk9SirURB5GwmyEj8jmekdPCqOy7cQQNdjuQW9qS9mHBSxS0BqMgcpDW1AQtx9gZ8OF88ypb+Sby/9+G5N0HbzX680z2kMSRv5tMJ45HYr2EB8FHeon/PmTQMYXeBInax9mbgbg7Eawx0+ptc+oux97lY87dp7IGrqax9qUgWYjQl7WBnbQPhirvafs6VAKFqc9GS0P1feaQ6BbWxrkpx12Bk6827ye+3KKkdyjQU1u9P0LpflkN4LxZIyaVFeWT17vz8I9a0ywIlNWkOFkRcxyhE3vvczIfOjlllQpSKC+kPtBjunVAhYa9wbuvWlPkq3rzmJNkklGCtI/hVyACqCzpLmEwgF6DsZMonQrumbaL9cOKwpRltMegvITce77wgep9gM/i7Vq0tQnuBbWMeuIJFTKr3cuNczXRm1ReXQ8KxCMjnMGNnkYaScqe+a8iI54htfoUtBnyyFnTco2vKGG77RIejVUZZn4jLu/oOqY3Gma2t/1Jtib6RU9po/ttLtPEukCSFh2x/zk7zrFOjyYBrk4XpsZxSoRgFCj4odGyiC06V35m/OcEmWOaJ+5pGWyX4f1ZHhxcPg/vp7mI3UyQBNhSJQLm6zE0T3VwbqW0fA/YsVZTUGkWORszwEF9VzGDH1qh6iYskfPzbfAKzFhu+MzAvWA77nbVWTGJIfSlsNKTYl7n3jmuhnbm57CH/UshvWfYLSAmAFCxwxw8TzLEeTu5ixkENblYffDhuyKkN/BMUVdM/e/MT7ePBU/SF36EbFw9vxfUhfh5QFpF/vYZVVI33FhCzQEZjwZ3Sm+KDSAM8EKZVzKlgNCL/gOeiiVShmZDl+lOYyNqkMV2SfyvkTTfkUY0+1YzbRtvj03CbPr+PZ79iDEH+/NZ5THRahhXB9cfKh34AbITeNieWipUX3NoPXZZzPaMdc1+uCBITGN1KlCP4qqsfE1Vd2oHUHAdCkeAxs64S6GOtQf1Nt3EcZ336YfgQRpAl4lzaWR8Eah3LV/ATkmsfoKmpZUfSmjdXrh4o6yDf7sYy3rBAf+oEJG1R0I5LwMFq7mCopw7VMkdEst3UgpXSQ/gh/mvjgUWlVscF6O2X06ZXW6M0BfINvksCmEYx+L5UNgjLqwwNXbcHlCB7IbRZCpHVxwtKPzrgoHwAKLuJFdmz4LR9t8FgjKVzyGtlgz9i/CG39InQwgSktOspSlz4YolXjQItNPvz24Ws+yaSlxsnoVMtqU5FLlA4DiSryJH5uDmXKmCG51PVjfrZBFe7Jycc9+dULvz/L+i0HRgD", + "page_age": "4 days ago", + "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + "type": "web_search_result", + "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + }, + { + "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDNwwmNR3sS7u8hviZhoMArTDi0jrPciqCsyuIjC3gF+3he7OPDXnafYusA0wtdtcP6V50y3aRyxKZ0x4PjtGINoXWxZN4j6RMr7YcH4qjSC+RvMT2Ic28yunEM2pJQfF+8THMI53pPETRq7kp8DkMU79aqKlTN+4PgLlhjuCCgC8RNgdaqVGCfSVuurpAmDuzsi1O1tESidSe8gIcN9BShdjDdDr5P1DTq8k8CbUnBvpviNA3pgEW7/ktApPF6pP6faiziXbND0u0mZ5IrsxLvyXHTYaLdP73S3CzFEivgh61xK4ezZ9TG6dq61Y7kzr/mqYumWOKpZ+lqdFexEfqNZ6CtEhaRTlmNDz7d5ZySi/ZTZedHAPgkrFiDD9Af2r7wFH/6VI62gJ7YhZaDKHizliNWsqaAYucoNFNoFzdXoOF0CTStQBOqNTRViusgW+3kgMi6YwC8z+vdc/xU9ol3sW/c/ayd11Qf135JOeg80XbXIQ2U1Vfx2Qu5A5cyUYinfxIFb4YDae1Dj4uSOXzakP7HC/eGEM1HN+FsaGdOmojCCpCc0BY3Ao6Ywz2YllWe0e3cju/+ycRzn4SI9Jqmq0vdbI/t7bSUTdRQz19Ddke9IkT0JhXD1ATcTow7AShna32Sts5ERpiv4zgDGHYAqLOPptJdspgR+q94tC7ePWy0FaSgovV97TGQFYJeCVuWrE9kZSBG63w5TE0NjXPVfMBHlRG89IZbHFbDgxy5J2EnJfTWCMbfqWyyds6332uJDKISgmXsOYYFey0rqSkAloHHxpOv703NMafQip6b8uXsDyWuVXVMwV+RCxXxiX11QizJ8mco5rbuSpABsIWF2aCu2O5hR8Gz5Ajw4yZMtFFavW/wBzzyrVQkkjcpSZJSK5UatvkIwhTkgo/dxtlT2qtg1UaYGL/FvW1veFlwWihtH1Xyi+mlJb+NGR8GGveDYBuJMyE2S3KxZ+g7duRkGvkqYrPq7khq3cVJKIO/gWfR84pzHZShilbaZqBrwkmGkEFvn1jxIZnOGUTnwJUuKG2bW6cAzvD9xc6QwsHsNVSKyTs6v/qiI38uFu9OU6KWpgiDddFFgfpo8ooZ24ZFaX/Pd/tkDNaUMc0lA7AvyfZjIvhx6MyPkRb9KGJteOY9JnQiDE+hljrjk6S0pDe25nSLWsyrZhZKCOn/jb0S7HjsTM2KWe4xEuRwkJ2NcvlqbA3ws/vPovmbM7xmApj5Xb9s2dZaXmHJd3H7WsB+chcrwvmNzSf7z7ThA+wzp0VRQJI/k05QACnJqHyi2dRS927erN3jkZ+2batpsKgFfUYvcr7MFex+/qkASJU+mwPykJ4Lp0+KWd4prVKTs0vnbXZISIPVBOy6HQMXHnWBWueTRle21d+xrLXZX3f8HaX6oPTDLhIh7Wn1fhjXXPs1RqBxW4IgxdGjXHFwxl0C1q9Bv1RF7sogEFVjXDffBTJEu0w/sqhMGbjFkF71uVxzuLZQpEXVFQRtcH+uw94mnhKTCanXle5hkckrpJ+ztpFh6/cUpXm+CVyLtWBavm9NiFk3ZCXth9gyU4TVAHg5OSxWNY/5yYijFNgMLNWDP44RLseTlW1tqgbQYZVz/8PjKbdYoAjxPMAlTqXEIChH8WbMnx+g5UYPVxeVPfIX5DS+2Kt7yuCH4lcKKS3I2mgxsN57NvInnD52IFb/aK1UPAHG0Em29dP0LKdchZMpzD45EPyC1YkFVQyfFliLPKddOIL8A3mPrFwXGIniLPjbcYrrLNRpHtEq3YH4KtRrRYKpRwpJRmKHz4PE7iKeOaisWXbPseKtuzLq/Vx899L2gBwzwkEFsaFhV6G41xBCf6AMcc3oFwYqXAl5jNXZScWjcsKXB6BGQgxeoUNgy8Pag5X40RT825sYEZPA2+yhBXOvyPKS9lY6PbN0BKBX87/8+eVNONPIQlFNtPEforDTk0nRhD7zDhjj4p+4MoIYwAQZ2kma9wlOEcWJ7/jRWMlowLVWYVlGrkLKCrx2Je+fa19o1ZKwHBdrxm4hbo3/eEQhFb4jbn2jA20WML6iCy6LR/BOOaCdF/4ReJmaiwexCSLxQJyeihF6he6Y9NsYJcxX/IpQt4ic0rjwvfwjvfE+U4l7i51NeNxp0pJL6ntuc7RPc0qbupvRAIyplu9e2bcGP918Zu8orsZe4r8dMJk68C5QMAzMAF34cnvFXfKy66VkzKx50tjlucXi0pq8z08XpKoC6Ceu1vxGtOGKM/lsnBReMMhsJnbeQW2FSS6gp9G5Z5v/NGBTHQPV2Ds5MGR8E0mlh3AwEthFZnyCPYzr+PKp3/onPAQUhqZxvbOonAE6ZJu5p8g4buQfPLDxfchiP4SdJKHIDWi1uPTNlBx3C0BbYbcItZJnl6w11Rpzw6Uevy48CSDWVk+3zNjFbqjihQsymYwqcc2s5zs4wrv1MAaGwiY17ouseC4VImtBGPLTjlVc6rUYwHXBVw6NCwtPFtK+Rav7Q2Su2C2SjNB8YGkvmUMdeCM4rIj7pk9WWk+iINaGYGsdpCxdxWBd2sXq2S60o+8vLHhVMT/zI1XEhO/nju8c9Lw5Xb6KfmGcRjVLnVwpmXUE19lYrIoaUv4O6xrmDggSa5Ykt3nEHdxjAKpNB6cfOKY4Yaog+PbF5nKSQmBRcgsUSs9wA1IMJ8QsBYbpbpcEPpL5J40gqtMlBIpCGRas4D6T/N21jPCQSESO64lH5Y2ynBLqpe9p+wkrJ8CtZKRHqTRLK48jboEwtmIPG6DxhVQ2eY8dtJWxaPjJX4wHEMT/EOdd0mmMUYQOoh3QX4cpsaPSPcjng+w7XOh5cfZSuaWQOD8GsuzGuUB5lVNNXidovOxvRVTeSka0RY/WkEqrkPeVAQxugmvhgVg3k7xIsjvlyjQl0UEIVfzSGg2HVZgNfFS1i/5Cw93JVsMn+yKJtG7DL//k0u0GAOVdu+CYIS/PdaVWRHADf/We4vqk7XS/RWL7pDnIrjGMIbeiuJGqDv/bAZxYBi81uq/ZkOT7oSfbNPaCo0s5K+8z64zxzNOeHtZDm55FxuzJsmHdsw0R2+dsRjjU4r+A5TPH7qNa1hJ97Iem9ZSDVO8jJdg74HR6LxKfKo2GzXv+OYII6332YUhcWtYIUSRc8D48cE5jcYtKxeg4wNAFFVLiJEfVC1aUfr3d0gIKCEhtSrSVOaMW3dfVSYC4bKKvb1KMT11fuBWEx2AgfWUSffyXaBSafM6s3KbW+M5CONXmtM4jdsYt2ogK3NDWxd6HevFl+CfKh0AkfFlITWX2GKwz/mm+BBV8CmRQUiUZNvCHN5cNn5iy3blvmqDITtAxA4hIucdDAulCAFiY0fBU8BDP4KDClH//JXz+rdezyP4K7AsgNC72tzCrRFk/GhOQ5L675SDDdAWEihGzhq+ysR6r0zq0cm4MJpYV77iD+mZgqeJDkscYVrJ3rsrJCSD904sYO86x4Rv1XbzdQ3MfPXILFaPqVMBuGDzP4bkT/sGNP5KY5O2Agutbad3QgRWGvYjJq5j7f9XJE/O/q6vdydKm+K2oaU20SGjfkRBK0WI/2TjdzMQ/wBYVid2/dUI86pZADmXNLpoE518CkqmfYGMCtzW+vWemxIiLaEkn19Jj3nTzaimKW1N7hSevPrVYqLEH0MdjqE2BRcRGJULmrWJxXHnAX2+vzcY+x8gO9SjwrdRK1YTTFoN5zR7Np2mNS1zGPAGG0/CvSjhF6B3l5zoyc/JSKY092yUeZDX74No3hCCu3+DCrY/ywc/eeJc91Gzy2d0bFVq9VDFFAz7OxlHz/TwDc2z7roSlEslWNuh8mnO2+UZhmSBjvHu03KZGxIPkwUoHtJcpOMxlAm9haBMxwYLLIzEBKFUC/5wZBklaCGgazDXcvpKVSiKhomrPkgCeeovUYnJm41cc4ha6qY2JJnG4PVlWU7+d3oju0tqRzmzwt8/7g8NL+kIZfrEb1MPwBoZLHaJnLzDEVPh/4cFPXEO8Aa/OvZXTpkijnSR8hYf6XnBU35z8IuuYQt/yk+sQVeMN+oSw0dUkpwESWpci7HenT3akVMzXqGiA/KnVDs1DXBv7DX1LI0kBh2MZg5Iqwreosq/znJru8AHtvjWEeAjviGkIko40LKiyRnf/6e82ZrBc7IcksgdsoU9bBJFx3biIeOUy/FcwQIxhAM64ygk0LHO6QglNgpJ25BEONgC2UJDMRePc9zop86OwnOfsEhMvvt16q3pIOOs5Bb5fa/vCReueXuGZe1fCX5YovyuPiLBCSlNZMBAOb01HD4qK48sv7DtIHfQ+tY622C3vn7SqxCFVgaWYWGfG3NRjMwM9IyliS+uLG6KnTtoRm0a023fJ/Q30dsJ6R+/bpcmulz5mszm4RPZVgTNkya+H5JRAfMJunWHgdkmUqlknNCMoul1d+uXLUHnW36e2cYghMqkdUIOJ3OzTu5II+CKk/80riH4LyEKOuB6UdwcXiEyAeHaUp3UWQHgfaG0FisXtuZiZoi1VqQxDzIMfA00VwobN4RgcyxZnNG9JyhM5DUifMyxASYYq5p1DXS18ZcjoGMZopLdHb8jmsvbzzRuDtqnpEw8jyQhdU9VgOu1dBXavxiK4vLBRaOBLD7Sy7QhB59acS/4CZ13re7ddUu1P+y4RD6qCw2gp24ZAceAO9ZexjYxp1dhWfwePu/9CqrgCl4hFG7hMhzMtTwzOtszVaNT7H2DuhXMsxpvv2xWAnpPC79beDMPjbwBBadhMmStqAUaZSuouVKDOJVyIepjOlxs3mpnH4vNQXGMQVrpsOYMoVOqwnRegudsX7W9SPV+FJa/W3E5In05SJrNR+BIVjajLZdEw7BPsKrlB+xRurdN8I0EitbNzIPAKkB+EeEtIGvVV7aL3K5eJGUCZwBj19EKz8WjJ8vCvcEGBqU30xZj5aSWD8vlTBoQX1ymIlFLBs6hP54JXwkyUaK1vzGuQ9YlWO456C05FXE/PN9alWtPnTYN9dUL7STxgl7aWxkD0h6ghr3qt1OCwy22PzirtK1AjvZGCCWG3BQF6lR17ZcWYi1EzxF/xM7h3wmkmZqA6oTE2uujN/Tjog5hB4yPh/JfPobQQYqvBupAH/8p8jShrQl1JQTmQ6VSj6tUW7rDBpYtqmVyq46jtvK68ftD6+jC+8gCiCh5VhGFnVfMvwSsd7P/514k+LYGk5pOU3n4k1hXpmx3Ezi8Fm5oDLOu8orA34zOpoBT+vN6hOBrzyg8EowVLfZvSCqR5qYWpAmuhCRb/eKbKSzF07omXWpcQns2F0auiy9+ObhdjYgNd1FPTL0blJaOGNDnmLdnqCoxGdz75wVYqiIZbiX7+2UuEI82nw9gig/7taQRfKiU2GteETl0EAwNYsMsdNzfB+ImiFbRA8OiuslXYbbMgiSTHQvzGPbE5ysvhbkB+Y9HBdwGCuNryv/PwGluKI/xeBeJ3Hskqu9EnB5ZAoX1nI7CfygBf8XVKFMOXaU9NwtHEV69hwGykTqXVAapHlE/+0aQF+Y1cT+7pnNXwXjatVi9ADoKXTOmWDsWw8y7RgD", + "page_age": null, + "title": "AI SDK 5 - Vercel", + "type": "web_search_result", + "url": "https://vercel.com/blog/ai-sdk-5" + }, + { + "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCxiOiFekHC0+MvX7RoMvlOqtxzKqzcIRfyoIjDhmxPG7soqbnXgcZHvSGeQz0df3ps4/KyKeXwAskO+SwblhuRtkZAsJhvxY1Qz1HEqyg1RCXfzy3sED627heCMyoK4Q3PVRcVHwJktJnn8iN8kmR9BRwKrsQ7lSK7D3GHqsYCbCeBSD+2mSNgPp/f5NpykextjwP1G1I5kLG9QafxUBns1mw9ySbO6h8l9FymB58c7Ba6LyirXzUiMeDQ7AmnNz71QAj2w27lZjDrWRfIXrMumCGR131fR8zxgc2uifKVD54oHwfj3NsSZUEHCaoza8hEHnHlcptNPymIVq3bdM2Zs8Ko8fICX5prAT9gIO0ijH4S4Y62nc9yDf7qvEa21jv1QzijYOyl1DIxZoKmYAhAEhYCeMsJNqenZaC9vdkCg4fHxJZMeI3XMc+iUX8mUBqy+YkY5XxIV/IOZS7Cs59JHpkn0xSehCcPHx9aZT1jaKR6UyHyB0hu1OO6rBE9uNPX6UdsA4Z9hF9QuEKGzqqiglEIY+FpWny4PAFstSTm+UGeysoCwxVA930oUmBA6HrzRbT2mlh0vZ7fmWAoVvJhofvPxFBKEmS98g6dqRARUbxXD8I/PK57t9JvolNwRTTpLizFTlu3wc5bbhX4MHYF3kq2shYrbgswyQLnqRoA37AGeb1GTDcB+vgG2S00g96Qn+X4ixNaw1maFJyAONo8BX7ovpqONNLv6NKZw91M68ERoTA4/enJ+SSrEwtuVYHoff7+SaDDtf17GZter9g0PdFAhwHrpnJ4ccMtVjXlnOw+Nem/muIYRpVBr3WvYFCIjt9W05xVrMfFUwkhAyZQWPhOTj0W8bWc/q7sG0o+uL8Js2oRAiTB8fYGxV/YiiIO9BUlbPYgxvrKXwmXsBb2m4PQTyY+9LpiG7LvjV4dqRXtBomgATl1BucRViBp47ZhCB3YAJBPQmO6GbKt4kJtBZoQtWEq3U0oZ26pwLTd9EQpYUzFii8Igq22jxO6Tx6cSdRD47gx6FbUpgQmMQJekjmf2mTwZB/u8umQqNnpb89tqsIe1pKyln4OghmYVVn7Mu1IL3AlUq7DZowA2Z3spUY0O3gRvMqqyLKE3Qg+uLXaA0TpHqnzcZ0RDHCW4yPTvlhGedtKJmQV4WH2m0BUSErPM73tDVzBw4s/z44IsNaPIkfVEy/QOYxkc8N1Ifctzrl+y/r/o/lxahI/sftesVYkdd13oh2vy3d2RpfBGXQWPuGmHRMhPy2jtTVSOoRvvffT+A+Z02u8D34/Q8GjLWB5XJA+ktdN/9EcmXLf4WjAqTNkJxBPScX53HFKURqev9Yx9hd6OW6Gvtyvkzwrhkm9Yir0UmAp8Z64bG6X1hByWxggnzb/eGlN528C/zcAXIiz9W4kXmycJnaXWt2QXSZbW4ONQJTy0dstA/O+rWKEB+/u5b2Yh+o57RUZzUGW9M8KOSoerX0agWX9vUnOzL4ZlAv3AgM/i75gL3V/+Wc2L5pfXJRRzL2b9b0axLXPZ7GN6IJU6KhV0Y/s85cEnv0jfUVKOSQu2EuFlz577MR7A6D+Gp7H9KIAoRl6z49BTi/Wv3FWqYncXUqkXb5OGMUQ1Ro20A3fchb4Kiz1E1W96YuntEfyfBpsFFAGdrDlJnhLWOuBYmzCmvjG892f7S1sIrzkoxZVZM0kWE7BPFNwGUvnqqx1dki60Spk1+zNUSoNL99QdeFxbMydpgucX9RuP4YqKc3mUPvmCfiLaTBwrW7sHOxRwizbIR6ge1MQ3BLzka//94LG20yy02cZYkjPGef6tlewlWSRhAV7ujhXZkJeiClGFhBzc6IzauGsVTktEC+inesqoM3ytl26GHTyPp7bO/9vpnJ1LwiAjyccpKuzzehPzdTA9O/Id8tZT/QszeX7MpQQwTqjaOpE+gHFXvaJ6q1ypHS4vzdFJb+2b3VN3ZF1PHdFm5ql0nvnetcgOhE3hUeix5NGWmNaLkTSVS2GucNPP+7n5D5qfy1hmoQct75xiEpuaV0LnjHO94DIxyPtrBDVEgwe3pwzlAC6NBhH6n8zrCPV0vyZsU9/pHL/c1ItTTwhdA26FcN4MGyYpHTHEYHBEtNx3iNmdvUS9GeR2d+iXZ0hAJAsmp8GpQR/7XZmJ91fRT+NfDeE4Kidd3gpwL1qQf5mswB6fj57ZuDDvuqPu+EcvPyTAIG44aRvTyHufANznLGq3QGeKf10QeN3Pe899X8AXaq+MVhKTdRnNzwtnR63Xs4NOln/spOod04LOlqdzCC/qPrQ+GtHTvMH6/0APbHdGq94O8NwT2JJmMfF62hQh3cD9D0LE7Svj/4TmR+wQfVaUnuH3LAz0MszvO8B4ypZ2lD0vby9JzGaKZdjSyvp+HTTGNWZMwp2Dt1doGAM=", + "page_age": "3 weeks ago", + "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + "type": "web_search_result", + "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + }, + { + "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJ7EUXY9ZeIg68y4JxoMull2xOlYWmKoYcIIIjC3l2xcRJmgibQjrVVH0/EEySG1q4WZW5lZ2OJIEdGykgwR8RQnyKICnR0fga8cpsYqtRB/d4KgjPxAXSoVzQ3nAChKBZ2C/tnl5JKqB3vx9JR/bP2nQAKEg3IXCE9nDgfrYnilLA3k5EN27b46aFQ25GDCCCFnZtrCJzOo36YwGYhLGt3nKQFIzZwjDCFMwwRgom+LPIA492icli0qBx+Np1H+CURvmE78X15QRokJ1i7XkmygmnJ54K4TpFUOlsuihvy09Q4HOFtCuH4z71tIJ8fADvZK7fhNuThoe5FUXX1Jl6YK4PdvSfQHbrkyPJWRwONY2TuYDmcR0lT1wWVxFqdaunpSAl5zs8YBiQEvBzdmtZWVQFIEnPLwwBiT5vS5fl+1DEDVQehfY/eK+yjkZRAzyeUKTkam6k34x1KhijeqRWV2GDBmlpxMffxbMe1kw/TKPhjV3aHTWBMufRV2NWntdUPQB6SvUeV5ZNksI8PeabyLU5KN4SPgxid8JpVCgx02+0hxXzQin3ywdZ177PoZ1Y3ZFct10hRm8XDIlEteHddw0fH5ogDMglf5RBImlmmKp844a7qgTDbmXL58dlb60JtnTN+3A4SYic49C6HdRhhQFFqfeVSVd5BQ9hFuj58gzGFJkDFY2+GEPof+ZrsQRELLjBeLJEwlXMvCtyJFmM65y0I3uMR5KtL+6VzICvKTb1T1d3Dj2zkuIoLqrVlGLK2vrMCL43w5B5xL1gIBtr7hnrntPZ1R0tKdiyCUTjoODIsnfsTHZJIgNIQRHhRBqGPO1IfAhZhq+ofvw+CeBuVwACwQlhRu5vwqU5MFUxhrZcNuzPSD8UTYAu2T6OFlhlSXZkUhYIarZl14GCpMnfX5j3kHtjdwwbxx+VZB8Acq5D41iBpqvlLiHK4Yb3uW4P0DtNwKtebWwwNWoRDJCNUixaCHzjndplqExlH3F1b05NuOlq9MyyXyp+NgZ2dIPKImpZD46GXr0mQc+NnXKXmy+brdFlLdogd1FaVJ2TqCYTRHeKwSwlRHsah/WAEwohsNaew9KPcJbSgLMdfkcJFKeK5z8VfrLcI5jHeeI+maV4h2/n47ifjXQgM3wB89Jxw1B5d/owhpRdTNZW/5BdZqee2pj1bcvw+vk/8fdHy8odkdV70PcKyB7hdkX/53YAdn0sU0r1SClsbduqDXgR7HxLzYmxXpbJd+LpM5hvFDM1sOWF9oR9dJQHNCCthQ498kqSinkShw1pQxLLHtgB1kIYNgv1CTiGg658JVfmM2j2Vd83FBzMhXf9oMA6frXcDuQSCgaBThQoU/CtxLJzhXwZKPxBn+7JoZrqZFuf7q+zAn4k9pCVikqInIlqyazEfmZY94URwvUIE+yxP+ZJK20rCQyKheV+12n4BCscteGkX4pFftk9zjkYyuGnz8GwTApxvJovtodp6TutDPtzOELD9a20873J8HB1SVlNFt39uFdIu1lyVqOZA1fgwW9g/isuh2P/plaUkf6voz1QR/xeQEZgpID4cg+sDybZVnzEdnx+3NVPlBeuw9kts4nug4xcCD8cBobEfdkdOp+XrkiyLrq3Q5eCzCUydJ/cYxGcQSWyiiP4oJG7ty8IbEZKzi33gChlsVWJn0JqgH0o25nqk7QHOJlM1oeQWMb+QQxw8nXLJlzSKiCv/3YnyT19gtDp910b5CSyiIL+teZPiPG8twslWyOCoKilxLUt3S7tD+V3PBO+jHn93KWd+fC5NSZ62kXAcIYFQ8qAC6jprwOiQM0GupcIqAz/bs1G5LQZpNmaIrAsGksmpXBMJtZsQCUSSIAO+p/gq1BaXfefLz0aR/PP0f4+MsKCpSu8IA+h5e93Xlb0tkdWSqi7GvPM1/AEaTJqKQJpPdOhdvIQAKqxFs+g4iO/S419cLTMnYG8fJn2ODZVkRem/8oYcMEHYLuQkBD3LLWTDzxWLzdnwfAqsRl4e5/eXYBf2W2sUxwkAI9Gg7B1QsQMdsxLRGiDFv9puttSyLPnVaOVQ/SivS8IFJkjJS1nQOikgWq0LapKp1WynbSHI9ZF3AcUtTj733+oDx7vJArxl9D7L1WBpHn/6hbseAV23Rn2vK5TSNvw01J7smrlG7x1ZZJBwsse7qKKfM/lTtiGCW0HuCzMcatC+iMLAXAQbJI+Lu1GP/J5G3KdtDC7cSVpoJv/wGYsU8R4Vf3v7R24422u/iKMOrivetZe/eB8ubxoWcsky61n7Kf6/QZ5gwUPBBFoMSOlpgQsdbexlMeUECbcRj28LeUS3XhSs6o+w+LPP5VCrGBxOo9XzEupHwUDfhSbdqtGyJGmg3O5M0PdoffvFDyFlPu2IiNrNj3Ke48k9rJKWdwuGHryvSLXiAJKM+iYiWG+trste6cuzCjtV+KO4GCWxp6Ow9kbbDbuDXFPDw7KAAzKuVlG3fScbp28JS2zyfKym42vwXbgVNEA4p6LD1HN0NNRv6oCmUu852j37MkD7rIfBEawSqli5P+Vu815CJNSYRwkfJwIXw3Dc7lbHqoi5cqnzyCNIkjo5rPl9swjWgS03WdXRu2pts4oHkSu5skCwXES6cIvCqebozVyMAlYIFXvqPnSyKvnEJUXc90fuyM6NskP4UiknKT+sLTQc/KHGYl6QrdgqenLjzjRbBr4ww2PdFbPeujcp/RlecxGP8aeBe3P8re2jqTp1yQl1CP9Tl35fO/VbdahyW9xFETwRg+6/DAbe0ZcxeCe4BQxzBJYh7t3IlzcdcDd3sDGFgiKMSunAQ4JKCG3dciHZgFUzVaVRXxnW6P5cx66Em7sTRR8J0g2t0ypFQeSPHqndzPgESousYAgqRp1B1YI9NW6jVDpJpGAM=", + "page_age": "1 week ago", + "title": "ai - npm", + "type": "web_search_result", + "url": "https://www.npmjs.com/package/ai" + }, + { + "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDI5QuK6Ul2/hPgEs9RoMZvr6kOyVbnrig26SIjCKbzvRMUF6XUs02cYL0BpmSRn5lwuA6HhHvR9iXIHkHpZqyP3i4Q99tpqK+ECmeE4qixd+BsJgZw4q6ZnS6AHyw9pEnrc1ANbd1veujyHZK6qLSWp5KWF06ZuxCVf9Lj++Lmf+hOv58DJFDGkPMWAuyPquvUaAcUwqOFSjJWwbRDXe2gKdpK2up/ZWKefqk3esUs1+3RRsqIwshVc8urVYkkavtO/iUfQPR48h8jS/nPoSOEVHmxGVtpncVxfaT/NwnRxuG8Z5J1yviMTQcVGhkVWprBi5uwVFhJMKXPZsRDrM1fVDEQYmLpjEbcqIt4P8nDwGzopRR4DbZg3TmuDQx2kYw4mcRL9IxCY/IKYDTru3q1GMvU5vBx+b3LEcwAZ2eenIo9R7i5xalBFcH7taf6NSeIgRHQiJibXxNqS6RoFWFoi3Wg95WoleArqiyBklzs9fRLd9GTIPOi8HlA5GadUZBjJ5Ds2RhHQ9Cuev662cFJIlWtdmSC7m47Du0eptwVivpBYEi7g2GmLGWT9xRbiG6yHbeytEsyfRmrWns2jCNvvEetMkWBfcaNXX4fAmK66wtrylYt612QRmDKaIKz8h3ih1WgDYD322hRk9ythJ3Csjl6fkZl/e2pPrGd67MQ/6fcx9dU1TBweOyrcGJsAwCgLuetcj+6RH6XfihaNLWiAr/GFMPVpMmWgv4PbQ2Uknc8UnBKeIpLtJ/Bt8chdU/3PyPpAV77FGiYA0oiB69Gtf5b/wQaAd6GHl42C4bxi4TXKyLoPhEUIIJXX9WR/J0cB++5vuGtc+6fVgaOFJ/TVAlSOS693jTnLKDvrXhwkQFoF3F+j+Hx/m0C45AGBSfjXjvAU478opr48uDG/+FIMxrkvahh65YrWsKwlfY8c2wSDBx0sDGfjjPjJN68sKeWUi/RRfvgZZxcTOw3QWhRaDFTKLGyKPID0GzEkAQotfeMcQBuQtHmtd3nOXhkpcHsMxZEAjqImnno7U8G5P4EJP4DRxG3CZsymlmjrSM7BTJa/hNJhgWX5RQhv/N2OUNjP1mFJxG9UypG9XBbJzCm5q0p1fWB3lWTDSMYf0rgZQFlmk+MWYvi4sjb2i/6RwC8BEi4tpRS4aDynnuCgy2DgueFR11+90fu4U3cVLcof3lWgzyUDo70QEMEO7BFQgtt8DUXCNL9N6PhYc5cFjbpWspzGzK6dvbZP87Rq7787LQ9s9oRGGvmcctR+GcrHDsQB5YZhjJewbrJAw9VcL9YOwuRX5ASU9lA5EFewd+JV6bQ+R5DICC38YHcSMOakJdKLXE6x3WX/3aT5cnyxxUMAzJFFlzBVJ4LDKGbCpQrTrlfUVH6DGdSvYgcN46yqouA5rmQr3PmBXPkuUoyWcGXGYDs9IPScNU2gixxYo7uQBwAi8pHigxNUL+sePovti4B2T8YNITTxOYiTjlR+J+mklkpfMY8GgHi6tpzWSyIqwo+mo3whkdR6YdORLRJHCX+DxzpxYpALYlUnvhorCBoqdeK3jpkL91NQQoifX76fn37+6xHDfywmbQslCW04HHmUMKDNgM+xreBavnWJqDHQQuwxgWWvyZS7mtRa5m2P8UjOFKo5RZ9uB9V3bwdajdeHKShfOapysQTOpAhvptk6Pm52IW0yUsyLKKQaHhJv9eRDC/xpviTTiCRCf+EbZL+h2SCF7IcBYeOg3dLP/FFMchaLfHVEw0Yz0MwC4oHPYI5RDfgth+M5wan7qt/l7RBEYRzA3yPh2ukT0rRPjqbF46PjPoZg53vTpOH9WV+0xEIRrGFiicp0CtG6AoVIQZ0B1174rEm2MddaLfkqLHvO2DxCez8raIiZcK073zkqbrZp84XWyzfBIKCSSIyVPRs5toXWgofhVsiVIuZUn25gnUt3Gxq8WvovPnL1rsMd1ZoPlrA1sXgnxPELCzCSMo5S5BcdOoGJO0zPuWPL0hjhOxxodEp6vqy6VcB3APbgTW/Su2knS6uTI34fQC1tpTvTwJ37NPj1kjWVLFelx+NvCvWg1iVXTkoUtcFCFTEbqihtIADigaUPHz20bLeaqkIcQYocJDNfdQLuNeKf6EdbWfvDO00BcQ7g/dD3veGN2rTdw4TO1cID/WjQ29WqutFmy1I4WM55OZLaqSNNDD6/k2TSsZfzCUnw22iVJ3eTpT4Lwzqn4u3UX6cltjb+1R5iizU6y/UxAd23vIaev6MCDfe3p5qiuI3JAbQCAuRDAYwaD9Qqa8FHzdOcPvCmTkIEZ8C4zzcK3ckcHTmZfs19MDdvGeB7Xxa3Cvv7q6dp1o0sQToB7SzPRKi7cChvVYEPq6OATIjeV9fx53c6FqA7QuwbyMY0hjdPAdadHdYZyKlT/ILySx012VP7CWy5bCLEMOmhVSv0NQ/9yHc92YmuUZLfiUIKC7i9V2i2nQXW4dkqZ4tVJJmoNUsu95H7YfLwetGeO9nyzL19NTSoEv0XQOfu52hzUSQeZsYZRuOYg7o6LzE4rt+FyUkTKV94HRFOTRm/HH0EokjKzNl9MC9O0MLtXKVND7MM1ksW48JhRb06Yo9kVTHNHNZT6CR9hYWgU7sNqsdPsQtecRp3oHTBi0UuSFxcN5aK/Iv/OZuDT7YVwhzVEKxn+C87qBRHn00ASNXlHUFjZlt1+XOQFYS1lLgzQ6l46poBbGT8lL6EvRV3YZkLF9P7cx5lIeDN9VDSwys1SxzOhZpVce/Cpc6vvq0S92hq0imLtxnsEsh36D/9QrHugf16WJIo1I0XjcbwMxOWe6VtXhZejmJ6zPW3LhvPtF006/+UerGjmE7zqO97jkkRD3AWuOBOUowzskdTVzPj1D3CMx8WGaC2M+V1DgsGL5KCsva0n5YJANMFYtsZKYIIJEeFedi7onBnjw7wnMAHD1PkX9iah4r3sy0iwDnj+kJF/C5VKy6aYGQXJ30HBTpka4gUmqM2tyrpH2nh5R8UxdAfyKhfAg3U89ZIa+e+9XTkv1UV9ZbIFi7UuzAU0p+wmv0qEDHPTqgUgB9z8uGZda0xt+n2MN1n9N6cMb6GDhpC1QyG7MQSgbud4oIunQ4DgiF6bmvoHNer/dLEzBZrxJShi/PRwrtDzyYm+nJT1EvVqBeuhwtc/c5kGYywn16iL7bA5DmhGl2jG/DKNgTuRfcqVmK56lJIO7ISkttKi1IJcvlo1d8nKu+22BZufRikLmVh7Kd6nMS+o1cd9ssfZGipod5E35AkRLIll4wpc0zPb3GnSAuZyTw26lRx0XwMoZCGfVJyS28EkmrhAC4FHNSTkMD4ERsFcIqKp7C/3TOvEtjMRoPH7cDeBOgyj9YbZJD6y6zOFK3YDkhOgoN/KcRX4vKpGv85kF8HIMniRhL7hb7GfR6ASvrHnYXQdP6otUcfzGycG4G3K51NV9qgr7Z9klF9+Wq1ZYTyPoShUd5hxddJ621g/i7Lm2HCd+FI2Bfo3XSwi+DqyzjnDJvs9M6K9f38HTjlMumBz6I++Cun/fwG+0F5XvqFgu+F0kov5+Un0Nrelj+FZ1fewK4e7ApjamY5Kkt5jp8gHOglx101TUM3CRVrkHurclOkTzF3h2Nx+rjHUEAslSYhBuOA4GiyBNyhEvQGl+vQYr2PgO3ERREufR0X1IBWDr7aTC2xZrqKBODB8EPkiyxdvmcrMvc/tv0IR4NjhYfD/LLD9tdqQRvJ30kpN5s7qQTMhgP/DzDKGfxa8bHB5qOlLq4ERRBE/qDvJVlNkJpSXUMA27jVCQsIsGhHw/RNtsXjCl6fvl3Bxpu3emibMOHm4deUrjtWUkxpXsgYbocBQvuosr19X08gm/A353UhUhGF87kFNeEJ+b9EOm6M6zbXNdlIfmrU6KG7qNen8hUfYE0YkfCjZJBRLC1Hs6aCrczVELTIDdO3Z904lr0Mep8vd9cvWthvbhQzr/Kz7c2JzHoNIFdHVoiiHHvqllkmSa/uegGw3Eecmgnvx/YSua1O5LFhhJtBDpcYYAw==", + "page_age": "1 week ago", + "title": "Google Cloud release notes | Google Cloud Documentation", + "type": "web_search_result", + "url": "https://docs.cloud.google.com/release-notes" + } + ], + "tool_use_id": "srvtoolu_01EzUEcqxFau5oLsH3dgLhiW", + "type": "web_search_tool_result" + }, + { + "citations": [ + { + "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAC0dqze+RTbPFWgcBoMh1hKOmIsQigZKu2SIjBkmz18F4Lz6rZMcRDEOH+QSMJzaHRsGNmUYqrNcYC/WbzhT09ybQeC/utv0msOSnQqE1shyPW+S7Psl8Jy2TvVNnzLikAYBA==", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result_location", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + } + ], + "text": "Vercel AI SDK released a patch update on May 13, 2026 with refreshed dependencies.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 256, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_choice": { + "name": "web_search", + "type": "tool" + }, + "tools": [ + { + "max_uses": 1, + "name": "web_search", + "type": "web_search_20250305" + } + ] + }, + "metrics": { + "completion_tokens": 80, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 16216, + "server_tool_use_web_fetch_requests": 0, + "server_tool_use_web_search_requests": 1, + "time_to_first_token": 0, + "tokens": 16296 + } + } + ], + "metadata": { + "operation": "server-tool-use", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-thinking-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with the number only.", + "role": "user" + } + ], + "output": { + "content": [ + { + "signature": "", + "thinking": "The user is asking for the result of 2+2, which is 4. They want only the number as a reply.", + "type": "thinking" + }, + { + "text": "4", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 2048, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "metrics": { + "completion_tokens": 39, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 49, + "time_to_first_token": 0, + "tokens": 88 + } + } + ], + "metadata": { + "operation": "stream-thinking", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "BETA", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 5, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 + } + } + ], + "metadata": { + "operation": "beta-create", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "beta-stream", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-tool-runner-operation", + "children": [ + { + "name": "anthropic.beta.messages.toolRunner", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 607, + "time_to_first_token": 0, + "tokens": 663 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [ + { + "name": "get_weather.lookup", + "children": [] + } + ], + "input": { + "location": "Paris, France" + }, + "output": "The weather in Paris, France is 18C and sunny.", + "metadata": { + "gen_ai.tool.name": "get_weather", + "provider": "anthropic" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01PWZpYgBWx2yCEBqiFwwtRW", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } + } + ], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + }, + "metrics": { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + } + ], + "metadata": { + "operation": "beta-tool-runner", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt new file mode 100644 index 000000000..56b4c8aa6 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-tree.txt @@ -0,0 +1,775 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1415 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + ├── anthropic-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 589, + │ "time_to_first_token": 0, + │ "tokens": 645 + │ } + ├── anthropic-server-tool-use-operation + │ metadata: { + │ "operation": "server-tool-use", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "query": "AI SDK release 2026" + │ }, + │ "name": "web_search", + │ "type": "server_tool_use" + │ }, + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "content": [ + │ { + │ "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDHDiukuMZ9P7cuHVNxoMqkhPqfj3Qtd901TkIjCiXbRiwj97e5IHHCAe9toqN1n26qStXg+l3wORzAAO0F418d24xozFzdqBatJOT3sqzgjrhHdfBQq/waT5zq/ue64nAX4hxc/Qf8+wGfW+EORQzjyzLectH3kgwzaC/PLqJ+SBqunhO+tBWaDt8qzYFB0Iz1TMsLkCjVk48M/aIvwSlqx7rhgCToBDqZUdKgpz2yJo17iELqY6FH7WBiSrpWNBqn5yWMTrwTp9zMA7MvjZCukQVDbxiAzpB2Ms6oL8YuhOwQB9sPbh+jrQ4zpQKJehaYp1ZhNU/osT3K7fUktioBhY7VfS3ofzSZQInKJBRSVSpxxWj85SH4GOAjSMZH1QpJVswuudAp9xQ5KSwPDEn9puSTT8tGdEVVoYHhwKfMIhskKuynAj2924l4trb05oxOOy/ZlTEsYHhcU4JdO0VDouA8kmIkOZxaVuWDQoD3q/L+JAkLk7BAYDItxvfPnMqoVzzgm/zZxF5o0G7tiKBpoZrIMlFuuiFYqQSlln7nFucjL7lJeYAiW0UcadRvbC5xglgMmc/zIPwP8pjdYKnGBtJF7d8MKs5YUXOmzj2CEW8rJHdic7ugCAn7iwEwnldPnUoEirRwFGuIeE5gc/kvcTwxzCT3J1rRtYGYuCZFXUWke9pTKhDa9HV4pBKAO7XPuXRt+UFjZGY1JGfbbx3n9tsdd0DdiEoMRInIxO7BzO+QgTkHc0w2NT99j02XpYiJU3RzGM3SMYdTtG8bdOl2Eb2dbA4TYuOuL10/eMeuLad7YHWtyXgt2Xd3flT9HyQHF+rrhEUYY6OWp7wecZFJF/2N4lZ+tWPhzcg1eUwImRU+ugmed3jceaGQH41yLhAwVHsQUs8Qouhg4zHjBfJbYiV4PKTwvZch6OaEaia2KvDFK3LKTGHdeuqqof5AuUA0AUx4hFn1zFU4EGE2pVEuCoscHhTorpB0B3onRylj30Buj46m6zJPzEt1gXMP5Zm5F+ZX4xbusV/Cx81P1lkKIKU5ebnCdjGoPIyUBkcXWx0AhQfXLLQJ1rvaY+yJrlM5VmnDEMN2yEhQlLoZqBTcHTRIYHdBw4XadWGGNiuVsOCgqNGnnBRBXDa3YuB6NT2mTZH7+geX+seSuO4ySBMr5wQt7ER3FiLkODM5As1Nm4bpg7C9HBBkj3i3RJgohwvBeDL2BsQQYxo0bPK/NQkeLpsoRpNqIgSOhGJ/ftEg3NLj8W1F9qdtK9Kezbzw9t9GhcRtOrp3rOwoZfDuCHFSDAFd8Q0TGRas2SCp6HDK8WdgIAKvV8oRbUNtFt7sE84PLIqXQlhjyw9ccsSqJc5U2rwuSd+J77Q77W9Ux/i85cMmxlXFOkyX/dOGrH/N7fOyjPSxA9lwx5r0tb4DPdAB6oOOq/UhtCHDNot28Rzy1tghv7ErlvVAQYR1b60qdI7tCynmk0aKPWN/UwHarCEn7tVsB9dKKzAGFINTJaB4o/fSz0C36/Fr189rqZeuwoVFttvoqE327LzFEM3Ud6xAgMU92T/4wjQqY+ru/+GAM=", + │ "page_age": null, + │ "title": "Releases · vercel/ai", + │ "type": "web_search_result", + │ "url": "https://github.com/vercel/ai/releases" + │ }, + │ { + │ "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJQcSzRdLlxpcbdoxRoMOu+o4ZkIjn6WpbBKIjAS1xrFrHDyPA3EmIrhkuk0f46MTAo4OQhbQn7vTNvFm6l9imrjWuDMdVvluSuLhBQqkRk6yWRRRnepqM+za/wU05/kz9c302G1sKfgAfnWN/oQtwKLTAxsomC2nKn67RqAxaTU7L1ZkvOl6/BGcO8ALZ6hJfpfFW2kOpq2BofCIQR/WekQJ7rwytaim7w5obLH7YLLDXG7jYYlh/9jslu5+l1L4+He14PpaW5VHjZNwntZLe32770ZjTA/x6yYUUWRdkVUd13SZKXlYt6xnLU9wZRHQPQtTK4R1WA078gEqzqf8rQg5WB02e2iKAHzjJ/TcxlS6QSC+LVI3y7pCVUFcfFIwNdzWRSxCIs+ndUSXfz/VCfOfrliOVWXhp9qh6vIZQlTCyJ7Yre3CNN3DLFugjckHixFQ/COKNESY/3vSomxipW/WHC24HU4g+/25pFKiJEQ2PFwdlvnt4EPP4d4icSD/xctx8YOrn2ZgbrvD1UHJVq44XrhwfKRt7tdnwGh1THc7RVfE0dLrrfIPFQHD5tWSN3Oixq5Uz6trMNtabXWmvhGmnZw9w0l1IZ7gEYcNuOL3jjL3QOHDsXKbqLTlxfbNi8guGYQwUwpMMGkVdTKskwoJ+ypVqriq+y6rvansnRK28Lw9iQo4s93CLyVlZT+1/KZfs/j0Nd4WMroEScAQsd3WofM2IA8HiZWWtxPbQs98P4NqGr8KLoIq8McXIDvIEfagtXxkOl5rrR+lc4j/ll8B141OI2CncUvtBQbhdfCinPkxBQZON4UldgwylksEXNffSub7Bb3i++NTwlJ8k1rfTMtgjSsl3aqJWEYRthQkdbOswF96E0md1BXLIXXThBrqeJmilUUIDmxZsiltL9bAIeAg/VK6n3apgTIQgbh3otX8gVqn3Ozj6wV220/S1ZPbl6vAp2+8puvjZmZ0v0q+ZNCMD0dzOz85TDpDB065mX1FzAxfGJP8lGCqZtDDj+a5Mg4dctWjLFILaErHHxlijfqj7kJFl6BETwV0L6cEVHdeVUi06PnlWn0aeYc2DLTCN9LPcuoK9bzFPktFKaZFX5+s9ZghRmyjoD7TwPifypNFH3tFnDr0rbrLv8zZIDrLW6zp1fTAfgbF47V7wWOL6quN/ckCkKx8cyrpXOuCl6V2J7hSLw0TC3K7U73FrV9Yfq0mHgfqjbY9byNr40FXnYYRtbE2SI0Wgn2vwOAMqrAh8qIBn5Z9OeQc2jlUKjGIGI7HS67CQ4fV43VmjqOYtj5WpoWNmubWKo4Lu4s1/grSA+1vLMz1JfVhoW99qqPNx7yAO2f1J/K+0P0mtwWgWtQQ2sGYvutJAeA6wXIvO4Ms2T3xlV+VPZVebmdZdqBFxGKH7JkIJxDSdyaDRNYeDviU0LEFC5das3AJYtuLuZ2CbJ5luCzlF7OxMjnnYHdWQBdwij854QEpohofESC1/SYnZszB0NqFIG78ZKwBn68L2Lp1PcqF2DTMxS/eRrxcTMFYO5XsaEVRw6kFqSqnEyUFHnfmIrDi7MLTvMqZZ4UYP0S1SwOU3FbrWVa4VAzhRC2TeHYhqCPo3mA4G4X2bnuzGmC15AzTsz+FRf8RyQeyFFlI9N07uEadfH2iI2/ZwSuWYcTl6BUQyvCzG27esJe8a2kWYGYVt2GY8xjHnJav2PUOckD/9ls93CyUQ9tqvYBb8YCvPHKz2STpV6t6Z4wy1WbxhdZcXHH/FLHzmBB5s1V7sR9Z8RrkC/R7CIO6a/wUO900Kk1J/hxd5GrK0nC3SL+AoD+0fnNldtQ65cYlI55kXYWl17IYbrc05tUKGDuqtDcUp188F5oA86qlvCBqEnK9JeQUMwIDPpNfVxxXcaTbqpqhgVbHYEB6YLgVt4oncX2tvVt6W33JLx+Z1Gq7iOnPLBfuhxaKkSleKoMx1Y9UWRKciL96KpfcxBFGHL1+8XNd/WiSt2e6+CLc/LvkzTcThbuz8l5XJGmAozO6EkXvJxEt23oNKylotFg1EY7cWFA1+GlYBAyE66+nkWpNCbb6FWnWzI4ltFgJEcELldjeN27moBmrw2DRmtEPN1RXpR3eQa19FuuUDM+Rvcr0ziYlJcCtLsom/76OJwxLk0USWdkXwsl+s9Y0z6HD84Gxwq49A0KsqjqvEq3qMcRnaUZOv6cv0w2zXBCJwdTjlwCpE9tuFkDcIeRP1JrAbtAUcVETkvGaXI3J2eoUDok+CN0xslOC1S4j+lxI5ZkFGaZvOrfX8y4mpDr+jKKWAGZ/HhBdg/6i9+spUdayS1dfMAbQK5xNXHsJILyh4lTIwELsl6BmrtOllqC9n63CgOL0xsXb6vshMN3Cu0d+bvugnTILZwUpJ1qTcrZ9LdnwCXwbEB7+1UYno8nHqfDTIoa6n7hWeQxifEE6Bp+uh3GCF9G71509NJbJBPvfwj1KfxGJ9BQtE64o/uKTsXidOijBRKZ0/uyqY8S5KPVohT2UIdadBdIF/CzcMTCSOveHxvqFOWTJreF3tZt4Y3zs4bSdsIt6x34fNxmC46XmQsjjqYDIX7fD0t2nsVbNM1MUj3iKSC6KsSH7JOCo09ckZveOtbgDwZKNTNBrie0UatwOX1PmyFyvMn8slAcNKhPkRpGf08nmfs7GUSdFAGDpKCt4mTMX+LXQ7orbIyAoBY8TPpPIrjCoeYjW5tpY4zW8WK/SYk4HNwsdsIrVBq6d8Q17P9m3vYfiFMpYeCpwt7QF73wVVXHCUpvgZKu0GtWbkUSv8g+iDEERKbTTM9uCA+SlG9QIHoPKqMSbleDsw/hKt+dr6erFgoihrAOlnMJnM6sZ/M2lwNEIkAG2TKmZe1ujuoxVgCxXj7vIoR2X7x3s+eqS4fxjPDniPP7orYUDctEi82f5rzksc8is+pP7MKjvzXM+tgjWIEyGmlDpETPL7+LcSi7SRSc6nBBpmMe+KypIbXWCG1fz3Y4Ca+Z5mNASEpEu4qPyeRz/Y9rspYV+4wemfVy6OWKfG0H2/VUEW6vGjrunrQ2JOw8pTQ47+dQ2ouplxv+h9PCT5YXXlCldK3EG4ijtX9McACLlHRdRsoLoSUNZZhhaP4wcOCbfTh3FrlLcqGEYvhW6hLIezcYdOjQNTvG9NjTruUySm2hOWo81Gpk4rULrr+VOLfGq8hckHWoieGgnA1Eg6gY5gNeR/3yVlNK2u4nuSgwXm1TwkTmWHIML/ppz+js/MGwI+8iEv2CQrrOSpD8ua5uCnlXRA98ao4znkqnaaOfjGVZ68SpPjhWFS0rm6kLqmq75R9EgnP2LNX3d7YljnsP7EugMpwITJHDTmA9EsxqspVORji+osT60Cnjz7X3HjvWh/nDsjwinwvyMEo9750CElFuslbVpDmzfeJsNPAwosAiQwf6rSAjxzjePi8wJEHWz8U1lwIAoDbOynJQ1950Nyp+ohyWL4ZpNf5G5i+L/26GmMSPQInSrmhTBl/lUXS8YFWNAaJshrW0i/4cIJY+EuEpN/41B7INuysyVfZIfZrk/Vqta+e3/KGYAfckUEkktWxFV5F5HTj34BgQZQ4ufwmDK1B+98T9164VjwFRFuQjw46uuRFCNpXOGxq9GIWCcYQVtzxS4aOLR5Y4SSGX08L/FH6Ixb+RoAv3M9KfSvbx2fPDXpvdfMsP1/kyUoboLaxW/q+A71uUy9WjaZZ+vdZFqqfZZKC1wd4n0D/yIlzA7DDbK+V2fY6GHtV+6tL9HgH/yuTn5ELgd7/YWu0m2k89aNgLsgaVP5Q9HItmUSCAUP/4797ejZyYqbJ9pJKhVLclYkZemz+YcqLqMh0rUmRpNg1jbn2iFJuJgMrkCtHJrcH4fthZEtJXx5yMuna0XEtrBM5je+lpXFFXI041uKotDL7TWxT0rO46F0mHOb7BRPB96fvDX+FyWmjtBET4aTCDmip125fwRyExI4zY2gPoRtvrf8rWgPSgh0uBbFxqIBYb78lVzMsZeoly9KvFkPJY6JcYqbSwPOcQzUa6aq/0QsldhRcc5RTW1o0SSBgX9it9coYcKNXbQTS9M1SOy+uGQ/8p6mfdY8KOFO7HHEgYLm9mnftOWrr4HbpRSoYS502DkPE+2YEfFXFGmsBbYvUaeVi9wrv1JBhtcWLWhSgouSjf9tknxUqfhqtk0w0hRhPINs9hgrLQCveiF00qgXjYfc1xIC/+8Sw2XIzSDLHTs69vXMswamqBUWNh5BhxeVFPxJCZ49enikfeiGlppiXZx72m4O9gvDPmIjU3aEZ9nISsmZHOtumn23oWFesDeUrrI3GJQnzykqPLPVvFOQejHAR7tvd94DRllCQYxFz5q8bGUGEsV8a30uUmGAM=", + │ "page_age": "2 days ago", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ }, + │ { + │ "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDN7ggpk1Bol0KuQEQRoMtSYSTWqpxJjaNnh0IjCV9CzSm/tXF0i+JKVdjFvs0lGTHHaDo233ef8+oHDqflzyiUAIBrBkozCvx3FIlHwq0x5TAEp0o5L/u+96ntWm4FApcTu9gGX9CxxLV+QYSe3P4HrDpynlhamCZP+VeYsMkk+ZZuT/OMF8I00/idY2HpAxZIU4FXhnUeoysXNh+zur2gA+FXR7JiOjUpOLMvA1YWjx0QpaeGNWnFrZ4cRC1yd28/L1m0MR/L3EDl1eGbV5IE2wAOedphYvDVIcUs3Gl+RIm+Cjy2DTvHkyDgZcW0fYgVOF0NSomDcoR7JPsMcq/UTTJ9pw854XSbEiZLi7Rzm4yuQuBbJrGyNqw068hiqNDOQ6hyb2SkMSKViIRkzQoj960QjhFl56aQ7GKP1hzmrHoTtMVqArw4MFbKRuNoaGmkBoQrzc/DXUTCkz0wmTMEbnF97HleuxOsWpjmH3tziVFMhZx1FxAb7npPQKqMxegNpGW2BYDqWIiUxIGL8yvbmPSVveKQldkcp+PUpXrqNWOvAhbKwdvaiP/78nI/zMNaQ0TMm535JdGB22MbZGtW39GwapEpfGFVoSElv337AazTVsVmjqXkBm7AjEfSA84UsS5Z+b3oMUpUk9WiJTg9ayS49CC/8Isgr7LHU2cB0AoB4hGtOg6X7k7+g4/ZD8CUCbzWnDvVefac8aU3EIC2ohKbkDlK2sP6fAmL6REgOL68u79BOCVYuiN4AtZs2t9scKsVYPT7keLYxVuZ3y9TlhaQ78esOEM7gXhUubqEG6sp4HXluUgc/IWrLxJYOifERrl7TulUUkIAGPG2ic06bEpsuzI8UlEfvdZuZE9qFT2i63Erz/UmM+hBaojWLXrklJ3o+ZfOJPWNbyeZhBW+4C0Iuk8XtuBpTNOlG3dmvNiL5+XlGBAJYLviQjGvOEucaJxLjlF/3PfrKG/RigUVlZABV48PotUj0g2W4LeivwRIEMbCobmMd7Tfprge/PqX9D1rlNqBST3NUJZ1nDOms6KLziMOf4/HX8GkUAffBUsSRVTepYjiAStPUAhrfQNevvQdm7N0iVm0cJ3c+BwuGI9x7Xdk+LgBjfATnMjliq5JzTkRqKnOuTi8LAHdRojbDvfBAt6MMViyaSIRPAEwhmMnjW9zJMdJgTO87PzBEL04vl3M7terqXYjMo5x66Ujxux84X9X4k9xKZ2qgazR1AlkC5jNU1125A2mv9Y2pWb1sHuhiMESwri3+SAN5AWeBd9Zf0EswHpg+Da5Yzqw+R4NSvK7Z5JPKFPzRbCqLTCq6KGiNP4nAsZMuHrxhtuWRjgbO/7HVTXeg2wHWBXtaIzqSlkkLsOySnMtYXFF38HELLxWzDD67LMU00CMp8Hif/T7RDg734yYU/Ogl+kDMwR4SfuBbW+a4JQdFgmIb1Sup0Os7hQOjKGeALjIVxQ/w9Dev7mCU+rl2uxrmHjvBIlqP34iv9/JWPVzaQcL0f29pOP2xXtVHxiN5vVcrD9UpUvtTZmKvPI5K+dZhqmb60IPVfg9mdce39eSZO+is4LzWpmcEDvsuxeT2jnbzREmSUZVDpkukMcs/A8IBFaktboHUDlDiAf3K2Kmp6mCXn02BXFH5eNK3girnMF29qOnKdTlIE5fXUw5CLvObGDoZ0WNvHbTm/INmD+abWBfr6MWuyFm2kvfCnWrOH36ErZJAre84aIF4tO9MVp8KqPUHeXCyEYyGeANbH7vbo0h7pe4uLWtCGn4F2My2vWMgekrPEb1ItIQdrABVVYfBG4dd7y4k88G+RCD+xRFvBzWRbWMK/rMvHWDBO+4fZilFgGmJaYeJYDjRXfzAmQ5sPnYbTt7ikjAVD2mOMDdEra5By9w1E9Sc9/BobK7UtRfuTFbFzhtAFiyjrHKGwkr7H4QqU3nQFAdYvlacdZtVSjo32UOdajY2OJyrjWjxkJUtkWeSvGIDWC/HQUoDI8w9R0LyvBnzSbJJJ1eGDendN0QKoyAjN1Q89lWSsjbkez3CAtFCbmU1jQGwHcfCdm+60QeNubytmAXO8xXa7o+WMp9iglwuZzwd+fK8fOEcuPnPBPhWQbEkrJzKO2yyvr0WvqWpraSUK71HlaaBFsaqF8V6Bq9HBC+lz7ZkbQrtIP4dFUg+EBbRCxxmmAQ61S89ucsRjjbYgh9Gnilw+HEZusdX1EwXMm0eMhQPsew22TzjEFI+jByHs0Hw+g8157yk6hDBNJCHFbAObOIul+VEPoUVuMhPrdw2ZrLc4TPXVS0L59BxTmBLvznP2GeG+H7ZQlAxZzsOWcAJT6FlDvT7xvLdl5zrbz7nKQSj08zrhGgtD0mtYUJrmgcqR3J9B1yc74rbbfvVfUB/rviaIl5WjwSzTxCycVtQRpa3CMjzcccByvHVdCpMHoqGQ67kmXpdnD7Qr0V9jAebsiSzZUtZjNcdSNb4jxezow/4b4ed5e6Qk6m8Hy9B1RiuOQTkUK/bM8nY1f09YbaAXO6zDVJWvS4V3NtzcDG1DZXjXQMr1SQS/4AxQg+7HcC4GBAjBPC0HEkis7uaCv1pVtUIZMRc1hgIsZXkjOOyxa4BGCKB+fncpx4oV8kN2dTJ37ggoiH8rAqxsOw8oFQKSSNFbQr7WbobfVnvySnFQXXF5LAnmb1oDFBEf/vzBI+f+uH6kz883UIs7Bz8xl5XqkfGehS/6gLPhdUVkflRK2jVQWZA7uqfJzsjXhNBFGMz0PHSg2nJBttXzVkS3SmfDaXHYbO7gMIHldKqcUT08Y5r7VvUBSVdjYOwUeZf8AA3L9amC7P1yq7LGqswTAi4H+F6cCg8JV0+nQeiSU7G7N2eJviptfxgfcHHJArlh09EbNESrL0izKHt2hG5jPOvD6BCY2UXOdFI3+8bbedoraT2xC9//9JwZQ2dDUxokCoD6P45npfVj1llR5AJrdEtkfPOG91qO1Qfcz6nTN25l2qBJILoyRZxFtuMlE1hcOdLYjGhgyl4TdTwcamoEtfNfPE0H+2tzHYtRrfrSvKjGC4mWmX4+RS3yGXHlfAvpgd9Dk0TINcgRe3uCnxG7Wjz1c1AB4QjxhCSeoqmHL5uWRI6/oxPhYy9oJ0KaragFdfK5MDaiLIex3yGE135zFao7X0C0cCgItyhLcgZWVWc7zyL4Td2U8vitZC1aOIRFPdjAkxDYMivYy/C3OdfYzKy6hxQfRQhWIRnBxZ2vvNoXF7o/2QGHeKdFKw9wCJ+ZMjldIAtEgFT9yQy2SzfKTCDrVKAs1e9LVHHhVUBSPVGZWBNCQaIDDO4eo3+PhQ+ehlFOE1SDhaBVXO/XDaEMnICOr+6y3rp9YIY81G+u5UDRcPK2Vqq0ltKEYeg9oEwBdOsnVMzJC5CWdNi+7EmZIT83RdK7CTfigbD2mjJPGGPCutwh3nLzL3yZmhIuESYLgbj2MJAW5Uk4DdlE35N3bQqPHoYZKnRcZTEg1cOn1rl2B42TMoO6hoqDLTeF1IMR55UnNypqoa9OagswWRp4QRPftT1mV23+tXn3Za9UAa9EfkWMV/yWwarS9v0n0d8BwIf06FsBKclgMNVyVmUt5ZTW3S8HLH3KKJRJiX7+VsVAKd8PvbX0F+NMwzofKRFZl533PXianLPpogzaxd5jf/BaF7LX2O17WGKcZBe3zPnS/75TGMvIsmFOn1R5O35Jitgqv/d8rrWnmd2QcBWGixVRYShMThm+iHDC8tnGSEmf/t9Chc05eIX0NJQrVqLyIzCrYabdAFTBSszq4w+GaYgJm52chFDc5H7sybGHWbIci+ktocsh2msmKLidncSOSuALV+ZgCSXn0w7s0Fdcmz2bSMCNVHhMaZCiENToRM+W+V2UkrR0+YzCQ3qFNc7NrsSVAjWffmQVuRgpXUR/oIDgQTRFbC3fNzP4tS2I3sBu3HYFUQH9SFLAdU/+EXR0hJfmUU9u/ZtMQzX0jdDGID8IfU1DKYfSwQu4a6QHpSNhsPi0ipHeESUSswAL+XgShkG3qyySPF25bX8eJgfdmxrasrVeL3h1vHORt8Ljd+dNXTMbMET+l8GZge5B+s1D6iezLw+I5rgjWbQu9a21BObkQeNnBDOPF0QxUD4zYcx49N3gWeCZwc3xJKUMRduDyv41f4/7wqSRwTQpvpKTG1NhQeu/da+h6pcR56n+SDLuv119EuATA7d2dVaGY6jueJ/iB9brEog5CSntbJ94Jr6sRN9J8kOe86BmMwgkN2ESSMx9WKFYx1+jes9R4/HiYmCv/2StyNqEaIM0XBfceUrreAuJSigMe/aeyMejLz8a/9KDJn3ISivsJphhVwRyC3otlZ4JO62w5+3hNvuJXiyR6Juv28knTuAVxj1hqPaYtHYTr42ERcHjzTiS9mxmm166Dm+/qHQd4TaJfrLSWKCK8kTluZiec5VDaTKCg2qvRkx70JQKSJdPgcQrtrJcOey1GflIQXe7A4OelAC1Ma3HJwHkbVn2hmpq4Kp6reH09kvr6hsJP0AMEYva9p4xBXsJajPB72slq74n4eaTNgyWq9paScWgAGnTZLcZJSTXTJdBLSFA+YleXlcKAfX/MyHbPVOfcmmGeEXNL6rG4L9/BlW/Gf/EthVHcVcnOF9QhNaruaaxkfkD7Ri2DOf8wzAs9J5B7YRreNucQt+XuqNJOVKbDPfodxXyBFFRtb8CK02f4ofyy9eU21jQWPpf4/d1d+mFLxIdf7rl+23csCpkwYCGZWdyW9XVikMteWJuojavBZwxcXdm4CzWla/LRQhP1PhK9M3RvtBpdvNS4Og7B5FwLAVEZujtdBijmwtt955WPkz5U9D344UiiNWf1taADVexcazWwUNW/ZhQH02d44Xm6grRTCQITOAA3Pn+kRQPwcP27jPSLC7r+QM2mhw7CIXLAzAxet0kIjb9+hto1YUyEoloUehv9hq/Nt1eznHSF3SjbdA6vvBCbjRdDDjG81/3VCt7JVobls+SU+Etf6ChRkTZUhdP4yPg0SSyQDNOqxZMQP7OxAEu0qFo7T3il247WY9ZkN3eN+gEh59kH5+tllHzl0N9yqvoyD+sJm69xabdQ+l00T6E+j/7Vqy9TseMKDsf9j25tHbXKCPxc0DyhhRW0wTr9+IWXbG/ZLXiO/sJPuKi6zETt8xVnnEvctC0DZujSGLMapAtwyA/WB9JbQ5Q5uI582CEUxOk5gu6FINjEALyHMrpO+hrcVVLVWXTnqPZwTJNfUNlOfp3y5WbSX9p08rSDMjl//nDog1O/HXNeQQtNxat5IX7OAQevbMGe2dcd1O+zhgD", + │ "page_age": null, + │ "title": "Release Notes | SAP Cloud SDK for AI", + │ "type": "web_search_result", + │ "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + │ }, + │ { + │ "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDGnm/N74O39+A+ErmBoMSivsWKAdEnTWwwvYIjCHslw24d/EHxd90S2xkn7YKcnHQc9ApsVsQONwuTDFF/ri6zoZZkwaDe7LyiJuYzIqyhQ7Z3t6Gzp/Msg3z/gNP84RnSe0dB8CHSfuWUCOYQBvBrLj8CahOgfT5c2ivpcENOwCarLJFTGaBzDV52PyOZGxavtrdCAYi6TOuj/pAf+xK4A9fbj9aQIIk55TpXqHhGqOU0lkmkYztrT1SFD+/XoqRmm4iVQTMFborr+nV5soPS19kxU5yog6P9VEiD9bbXL8iaWrVXLY5qo3vx0Qq18LFTTjV+ivytO5aEKpdP3eAEmEQiCF9IucKl/NALzuogDy0CzdRxnBVJy1N0CGUUjUbl8QtRi5dFtgPQ1cp8gD5vI6XtMIU8wBdSrnihnGjqGGf43VOfrZ+kFNrL0JrH0BGibXM9CQy2CgQCmsuT1lX6FJAKNolY05yaxXfXbeIk5oF/sqh/JDNtUIapZekP3Q87fw8tIP8PkHkFaR44EPkKFU0q8rh7YDW09y9mBqZ/1zNv9+3n0fnD7im+3IQSZFiKKA13kMJ7cPf1aYkNQYLdyrKELyuZb3rxvuh+a0NqOzAJEzZb4nYQmHWP/QGfVuTsE5WUOfdfvBVBKduZ6xOiVgVC2pKm2cfImCLeBRAOlvXyypWn5/9BCgUIjUeV/kmz4CeW35G+qRILmABS7o95vJgbFiMSBktFirAMfCjuSg865kP0CEshPZotoihio/lTxIyOcEbegJtI5Z5fHNVZQi9JDJq26+MiqhYTA0AxLAeuziJ+6ypWrbKQhyuRyT4DeZ5KBSM6szoxSReXuGJMCmlxDiiJzspKrpnzfTcPffZ18NZaEuldtmKACQzorhMKYw35cVnx0QvhryAeHcjr6KwFBghs8qvkvu8Z8Jo5v2vHLPokw1LLiVq3M4mwgeRgvBDiyvzb7GegMM4UVdsGtnqLj9tk3Cj4MphdFhFILjI+PqLG+dIKsJth4yeWQ0wYpXX7fgMhrqMs+EPVAML12/V1GMlbojECYml8KnQS9vvHLWa1FY/e3/HO/gCFEF+t3AV0Dl84dOQ0bB3K2qK5SKSUwMJg365FcZpMsJC+tM6tU8DAoXBKe6U89WcMbb7to884Sg0pT88gGCNq3F+4S3JOd34VoEh7mL7RzLS0YZBKAJB1fnH8NZzxVGSagOa1XHJ0H0b3P45D3JwvA/mD95lSlRRSHlqgZsALGvIYB9M7QgnopZb6PL97nCm1YQw+RwFe50g17qhRMmbiqIsPoNFGNvUCbnQGW2PsxAmIte3NQzVB/L0cLYIcD9m8lQixWyubAWrwikLzyN3whyWscGYh9NXLnWpdfRxz/LbyrbEEAEge37wIXNKw4PKjVgSAkTRaU6aJwvqjx/Xdzz8HFdHzpqYDQ+L2Lb6ndv+Y9dT8GkNHOYKT4pmeBbVrGc9NFZrvv1yeUKqAcsJZj1m6NjuZIdsUFA4D2U3X+WjkkK1OlGXZKlXLvDfM7WBoWowiEzqir+deqh8zxQ9pyY5oVLWxJdhYJYKH9RLOzJWqMx2e55URbpFi2I0s6xvdJqSsk0npFHUWbW4Q+ztHYKHV2q6Cm0DHXt0M6ewOI+vJnykBwuh7ihXh0p6ML27i1QhGvQNhO421oSWQGHBxKPDfZw9wnmNxbRTU6flARbjKJFBdxh63+1AnBaEmu/uoLEyfmb2gK1wfcwgmG/FYDROWtkhcy8xGCiQROpezuFN9b4DFnXATlhWTZO95Famf8jb5oIADLH/Avu0HEa9Ma6TQHoKPiOTejrdd2NutfMayBZak/pRI/bT+1NFIoyQrEVvNNrCsgMpMY4g0l72hagkFhYaMvLC0IGLa1npCwJBAOvXGwGrXTLhxgVWqfEBdt8dv7WVCmUMwOsF5ZpS5i2tPF5FMPwJv7Iq5r7jaDaMwCEsakaWigmulk+mxjE++RUKwpGh+amYEo2ire4+r1ItQ/Xjs/xjMEQFf+NXKm4TjbxAi8CJHvXOtQhIb8mqw7FVz2TEZFSITOWktv5EmsqWnzzyVORuRj3ElyLCKc2t585inxfMpxlAHDPDE4KuYjJTqTai9uRDWMxillK0wD/ZatqEFTQIUbiyE9RKICjs/PuQnjoGahrTZX98Q50kEg+UTIw3r40VFohBfYiZ0ccHn4sbG2gBu4MNlZLvYVuWhZhxOy9I6W59hzX5eh858LGG/BP0Eg9gPTBYxCxtrioa26QbdEMRfquNB+kyIwIHbyaOV9zqfMp/fG9pASBe1b3zgmOiaQFIKr3gjzPYk581O3k+e302qNVpCV+MC5tPxMopf4GXkCHCqq8tJC6TflxAnlKJUfg2xkIhRBVhuKEVpR2NSrWoUsO0EiiN4fQ/8v+OfUlJAOLJjszyTUCzdOS08Ar5dkrFL2U1AsJYbjpCoj4nGYstrm2gdi5ybMzzQapay6krcHEP61a2vztc6E/UjiEqOgjiEM+jV+62e+jOPTodjBe5scC+4hRqbFlo6PS3+h4YPAK7vQMnJKMyYGNhN4jmzrshRXAkhAlvxHfLyYwwDlvWRgtZyV7yx72FJkvAN20FHtlcPjpu4/5Z3nMxIVRefCq14Tm8zSRwlOhDJt09qHMT8i03Qiapnsbmz2LiLMZGw43iiMKRIQMHWXYUl9l3Pz15yO2Oht26cMoqi2Pkjr93S3SiMzIuTV4haXZwqY3aPhhLLLW3wNUR4FwZbEzwvRi2sqHjo799q54Ds9kT9Dp0d7TNNVLoGIaR5o2APKaw7KOQAI6spHDRIMOouHAgzdnB4Jqdm0sZSQCPvcJrziTgyoTWaqPwbsq6VmDQB6YTXxflM7pT8iJCZBQc1WGk0XDC83UyWe7ix07346hf75Q9ujr/dSO7k5vRQ25oKKaWGB817drvbaX2APA4NFHMSgAhHxhBV28gxUeJA+dJsjgIhaI2NUxU7EHXbNBCM2loQ1oifNArc2JXm9qYeRH8eA0XsJ5JMACnFrK8VrUc1eY8930Q8nXKSs9UZ0O+7XhauXlUKOYxrhJXw/tODGt+VP+JrUI57eMOnFuhflFpUUDhW+CnvfeKt/m8wx3F8grOGjlESbNKFd5XcCKnwJAy/RtJGBijRXWA8mMGSodI0desBflLiO1xzRnL1fiyfuxbQ2sMMwZUor45kCd8zxe6NesK4cDgcLpynudXGcMwsjmc6OaUHAdbWx0nKJ6BdOvJ8V19dlyMdpphpmzsKF+CapxedCMygmqg/Y0cWHmVlPDDwYusSk1wHqNVMTFV9HVgd6IioTZvFepExX+sKO9/ZhHCRVvRNE/xXlzNrEgrZBaNVKt7uOkz64Kc+EMvPGIBE6lN1UxNyFMj33MIa3OVFpEoXgythtcRDBURb0f5FQrJtOiMMDwmDbYJpIeyFpEnkXFXcH8hsfWhuFgiUpuKbkMgTonn+e3IOyEnUkQoVWVAs75zSRF2+hadmI9swp4cUvcn16PuWo3JlKujVWlMneb1rRwBV/S9BiM+MneQqr+vK72UNv+EJ89c9v5moPmyDsGAqR/H3uDXMqZVtUhQIpbM8t8eaEKTHmtC5XairSXze4WVuAYAw==", + │ "page_age": "1 month ago", + │ "title": "The next evolution of the Agents SDK | OpenAI", + │ "type": "web_search_result", + │ "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + │ }, + │ { + │ "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDEwcBIM8v+I+UvRo2RoMJ5MCBFs8z0DuLxQ7IjBiQWOrFiwM0EkQ/t6s+ESWxJ+iR8WWl0z60OA2LfGjHRjckq1/jikTkJUVQ4YTGpMq6x21x42bX787bSmYAv8aZPHb63CDU+qtOPlCQt+di0PykcWdhCYVjpRHiI1WzAgz2ztKPCzLGE2tVHxqgVBwSVPGtS3uHblHH+m7oMXwPbx3u4NGUYIXc6wf8gxwtXAYt3Bc/13GeEmLd2hc30U/coSuwcsRLVwKoNiKnwe0uXHOCbckUGPxOPivfcvRzwOegx8Gx3I5sbCuoWQLUv5BHydQ7H4FF02gHQ0ca1MG4We21LBx8BGhGWgTve1DTxrvlTvB78hmGDBUeoBCAlYdjewWQBFFpFbl+kGGVJB25aqsABzb9hlgdUddmPjRfIh11X2i5/sA4CkKA3mzJFWSFqbfstc1z718Gmg7pMBscMKQ1s82pE0ll+onako6PuHgqaIzqpGmh0fZpFoEZqtud0o3HVnghVohjenvLlIviKjvXGXAXOis0dhI2Kw1IehXdicp7Cd5dJyVCGvjuAdG9ECd6MyOtzKu/srQlF1+OXrtuoCRpOevgjdDwCwZopAXoQUYJWNdJQNseB96KPchWA7lDz5Zueym4pSiiFpnXHw3CsK2aAiSMeIsZEqcKEnDbkSfXKDtpCqs34xNoJvgULnWHm/DbPknl+67TV1v13PCp3sJOHjnen4iNsDyg66RLkqvQ4Ufsb39k8M6kxIBa7ypmsyXUlx2VOnFs8P7O5c2XMR64j+g2fvw/Cp1G9EfeZgQIkkVvC7h/UeALGsmke2Oa3dGLDQHh8NKc6OmW82TXU0kLcmwnLJf4IiidCtckmsVeLt0DpGTb8ItjJnz2QmRoxTKEFTYLz+TphG2LHrIJSsJz+ZmK3Dg2ZQS4cPTD2jEs5EhGghglrntgDX+2woOFUqGNnQl9rIS7ClcTToWZlybw7YoRpq+3ZMeA0AWhI7irZoEN906IcxqalSu0y7fEQkDAHtwqmy0TkUUIOpGzFPsSYsQty6Rmxv37CFC9iEjzOQ24QTvRj/sjN50VO3y/NkbAFv/ZvaVrpkmI0VibOjr8wrDBrJhCqgeMMVa0xTmNF3rpOLz9Ehfh4MOIwmpx5x1qq1VwPZ4IpMb7ND0OH+IxsrFdq9PTXZW4u8muS44Ifc6wmTSUsUIlGS3kDNvT6NkbKOdqD6XuqNuYDiVTr2kkRto7H/+oCfdbNSN/IkbY6nR3xJsTuP46etZumDg13IZaV1YUnh1ZWlpj4suAhGgUq7tiSa3WFunm2zuCPffK8WxE8ls/eqi+dRrQeWZM1CgYHXbOPJeAExi+U0W88yvmMz2YMJDAcTyaFANO3elyvrzl88hr3oVqezW8J80a4TnQJ7ubfgCUdNu5BrIZvThhwvH4auIWFAFxulOiCp1hwfvI+tm5CG4YArnqeKE0DnP9Q5hUNwXLpMXAIGx6rrM8ttPXTLFKp3VvGVevB/mGf3PXJgzcCeFpwvaPCxBc1KqJ5BByuVWf6lMax/zpbFJATdomLibDGRkpyNgnEjCvLDfEt76fF8jJjB2M2nGYyKz3BkQWVWFl450S//crAUP6th6O0uadcLNe0NIA7gTF1CB0E4r164Hv4dzaX5yQMkSBqIX0V5WRfdAQ7nrZg5TUSq3wiS/Q1IGSEknzSTgvdjLXhgHIpN2MmeDb1XS/Woiag6YFHhaZstYo3bGW/g5lWpS2CbsLHHoMXRd1ZmxjVizAdPcXU+VbdUVoWa58POI8rfvEqLNFph1Ye8mcCb5TSXTNJlwkagl0UtB08E/D6gZLlpz+IDvMsPl51OIcbcsDMIqAjWI8egmlmzqtXsy53QGSj4x2SJAlgUM4hCfy+4vdWCatiotuVTm9zBXOFNo45pK3pPo4vkltUdNb/TADC9JyfFnQqMHhAZxnac+LacefxiiWym02PpgakB6NYVnNkVZ6ys8rBY8xJ0UcnOZJ/CgkWJgdt2pOy0zMVdx2gO212S8j29YpXwgVTrDIFeZ8Nww3qSrn6ZnJJOtSdYVsiD460mZGmN447hx8sR0B/45TR+/UB8Mq11ZCnsyWEUZKjmdIfd01aL9qxVv7AQieS7B/8TqAyTyw4khNlR6uVfpO1kGBW+z3x+ESy1BiwHVmxY+DuufZH1Ia7LDjQnt8ulTaqdjRMIxO1yFf0gUBBhtJNMsYjWA/HT4yxnXtr3FSHuOtqx9NDpmAMAeQkeGptK8ZbXTgUfHkHPQGHv3T/d46HrVYSREviOpXEjsRdMTbJ79u33KJSbN2/F5wk3nwqHYqz9JTqzQ+dvnQ7zKpEkaI+z4lMid7VyUEfj462u9QcvUAMBN98Y/1QI13J0tQ43nPrWJS92f/gjp/2H1bt7CJwUp6OsvZleunGW59NgQ3qIZRXjAVrNDnb5AMMQ8o6sIILpNwgcKOSf1UuklTXS7Z6R28q6lM+CRe7Hh5mYgcyg+0Zzow6psF4yFUdf1FkD5dsNXdQKpST6hRjE3nlHvJKPDukl89r7I5JapxkAx8kkOFNJ50El+gGTtHAPyuEahIqcdZ6x8TmoC3v7ORdKkxgOK/MxCq3FTxzRnEkCXhOiWkoyZBL/jHxkcYJJyN7oTudoeCmf3+w1QIvPbzCn+TDSgqxuh5hbwXrf9WELgBesId9diu0iWssBU10o5MCSs5+n9iqakzbB51oLi6lRlDy8ERroGEIvdP6R32XRRX1boNMCaSAfi+y3jKky3UIY+dN9kSEIVclvm1ZHUTYK41tbjZ5WbrZ3W6Uwu4eCSBBEwM3h3dx9THmymctMCjMRkkaNEso+r4eU21SzVPiQvTGh1mkmBEou2Q+r6cWveQHnza14/LTx5+lQbgcZ3o7qMI3lM8pljk5ZD1HtHBEKW7nplBUedhmsJ0u7zvd7YmGu5FdIejjKnS8rjyuEwZgsAFneyVXWlXguFzP9GDCHzWcZlyFRlze0+4lvUgkd/SZpZGJ+018HFIhiL4q4OntG2mT/f207XU8+TyKE13YWUv9z34CKKw1u3RTUD96QAFQw1z9j7fQJzm17zjTNimij7xjulPFLvzirOoA4DpB0G4rmYIskD4zLRRRs4QvSN2ps0rLChcrbiVpjnPjKZCC5DRuNKACcp77rQ2R2uL5WaKaEc/6CGNEStxNSE/8J539yQKXKoaIeHEK2Hp8pQK/eQqaETpEhCp1/Ov5DT+Z8cSFdiUF4x2CWP00Ylkaqr++l9OZxjjHpVcNxhIYY1zuIVuQpqbG1NnEq8xOWC0wLNn7HAfs9w/PvtSKzjKPFhb+rlJUkvJCCqN73+9qmWMFKdm8cOKJ/qGwF5/m8Jx35tfRTSD6aQ0mfj19mo80DdD8DHWeU7uP7R28Wypa61X3XzdHTUjoRxqNV9G5e7vbvVd+AE2/3pQTwGnit2OB4pMbMuOpa5teT8JfwefPEjUmaT7JneAFjCszPkaSyd47cDbhd+icWRqZu/U1zBJz3IjnT6SHetJSSVW+c/aTLmjjqIrc33OG2zfTvvpsZK18ipjoIX641OMP9HG8BGJHr9n7HWWt0akEJ+Js0Bz+vzukRTP5DSRQ7+XxLi7kaLMJ7hZN6gDeVgN0KMpQ3eMlkuf5S8OQCuHUzIyBX7yCYwonxUqnrSn1Do7bl6QgHiCI05p1cpKi/JwP59jvtJtrfD/8zvYaDBjgnUnlz3IwHaSqSSIN+wbx2FN/64U1BsUsrkMXsefluxcXJlHz/be5bb7/pDRl/DBGm6WEMTqN1Uh5Ma1I8jPXSd2lQq677uDi5Ewv6g/R22GuScyFFX1nk0CFSDUG4TVwEyfhiu1ZY4/2CysYFVO0PCvb85sfFga5fyQwEXpCwytxRYAGxKn7UOWQtCnxoMELb/jM+qPuPyMWR0oBekjllVCEugfisE0J4KYXyjS0Nu1RjET9YrHn+77Yvpn2zPHt6X4XMZy/KOI/x9gObT9JOX6Oq23gNHwJY+XGVj5hAuiTiqN7DWx+mJstzoEpsFCxUQF36wwwmuNp8Emhf3jA7MElmGd97kVwH7+/xpIpsQEOXZpoz4oezVS+orgAwBwFKSdVpafGGgz7LeyR8DTaWE9UgEP8FxXCejeNeMjX4cQlSc1KZX9iN4mf1zgIdagc3qFmYdLXDgsEYyA2nzxDB457SWjQBKKAzu8rQIq3rgsVeid449LHKTUVab+Bwp2/CczMF44WPhHXqCw+WDj6pDOZWlijAIFq58KGQLUVgzUSJyfFrydpiim859tABmPWIOuVRqPRzjEgMwZOOlTtjvljeDQFyR5GV1d5Zmi/lrfKshoXiSOoCZsDJ6VjmSxBGHPXCGkshJESzCMajhZGLF7owyHlnpWeuBgA8iO+/lq5bPnrSbLLC173BD+E8DxdJvAFLehfPEwfTKy4+n8QqId9acz3I8kL7ozN0P7n5tOJsrCPEkBUCuXosurldTnz93PEo69MGpQWUTQxKGpq5ngxbCuJuRALmwwqaY4ewzsQF4QiOhsnZfSAseqvEDa0E2NytIY9/Ak46RJ0uSyIV49dh40Rk09nmRbp1OlAeVEZ1SfSc5kn52p99qJi4SVtUgVu6xvJKc3Qg3JjwXLlWsUUN9eZ/79ewa3uyYeibujCxdD1Pnvf8/AyooBmvd4Mb/Uowb6SbbP4WqU7M005zc3o+261mqynH74sEsEjnh36+pCPvzfmv8zDEbRed8yKVJTMTGrpIf/lx+eZVA9vOn4SWCXmyy/g+ClM37ZLXbhTOcAvgUr/8NdZnyHvbq4IaCexdTlj7Vwn+7lz0r2H7KLB0JDPWd20BA6O0ikIVs+q4scq+Icoq/vlyT1zJpL7YmrNea84BFZ7qEFS8+XGL2DzqRU3ZvCIDwp2cMTyHRwo03OcCAm3wH7bpRQ+R7qUxP/uGblj9himXPJIxiv45+88QXYCPhl84oIIF0n2WX4ApKtNK9jXgLjq37r5VPl6N7CoK30etGYrHgW0NzvzvrlkELmgLE0/nfpYrFc28RoNRbz92JIC2Y+NiirCNIG0UoaKaSbn5U6p3yLlOabb6NZxZtz0aH2J4O9LBrBuIahI0oc0VDkA+0l4NN9Sk0jLosh4t8amNA+m8Ji029Y2z8l6YbF7O7rHb1xRnT/5JUJsG/iNRhzj0B60tiCgMf6/sYAw==", + │ "page_age": "April 8, 2026", + │ "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + │ "type": "web_search_result", + │ "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + │ }, + │ { + │ "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCcL/yWoxQYgZdd9WhoMpJWc2WxIyUJ2YM8xIjBH254X083nJiyC3C27Zojh6fQNspI5JE1DicaOsQNqmvDS9NLd4WaKzpIhJhX1YnEqhxHHXuh4QBX3gTONGE/U+AiVuImvhHojaqu68yyvU/x+8DExAKuirhJvaL1TfWQBM6VpLk7xYHvBkk2QJyH2m3zE2shkqf5vYYUgRRrs6CYHWbUhVEcF901e6zAOUqKVaNhFdJgsbUFNO5WIxnv7Qwy52r49uDaQLx6GwOUlBT2QKmW6yKMfTPPEwjNN4ee7OdyRuGgIpdPKplzf8USJzU4R7NU9GEiRlI1B6KWQVQF2OY+Ep0mhqqg4m20SvWwkrDCVVe5XsVGyJMewJR4uf2M6I+SH2S+zUf+GjfiZGbQuUz1Ae9FK/d6I5/VDBWy67eYg38on3h8mrI33u0XoYdYCu/7gJl8M8VGB14IoNmp0VJXRunsv3iejsKA1K4SKkNjQxom8XBIalh4Jv6tVVjmJNmaDFkqMmzI2/nwXuUT1S3idYEvTkgmWvQ8G28fer45zgbkddKW2qycKo15KT3kuZqaVcp3rbzSIyU2iblGqXHUm7imF+MbILuHH2ITEvKKAHnC3ZDp7JBA+oRtyZImNRmTO1tFF6q3BrB3xCoC8Ozm8wATQYFZS5TSOdlWgcDNshAaOlpzv1eAqlJ38OVIu3Y53Z6owfMV31I8Mbsv/YPSxlxTVN3uisgI0JXkXcs51VZfUe1NbMfjClEwF4ARqI45W46Fyl0i6MZRcB9628Yc+Y6pO8llKM4t9n4Lth3pUj9nVC+T0p0dZ0TT4FQvXVLok4sY9PkZpqoZhvPTZKdwqhlWJmmQFM36t1J2m6t6GplBFYyp/K37KWYrKC/L+ZmCWO5wkFMFOJm2Nct179670hrkXW2PwMFYJQI3IEmQQjE+KGPEzLFawNJJkn4GSu0jFAYJIwO+8Vn8EfKPCMZ3vDjxBFsbVVjEzC6qQcMzRnezvpad4pj4dYMHIQZj/Di+s3MRJrYE9uuOKP6JYMoJTZBe6LHu48MtRA4KPILDMyJ+m7g3oY3APVy3tmV5hv1KFnpGF9PYC0IfIbFs1m7YDoaiDA+7gk5BLpw+qffDmE1Dv4cpVQAKEsj7Id43/v3Q+ulutrfcw3lTmf7/kGNBiqgdzntdOWg/X13oPb3V/5DoAYSCNjrNXWba6p64Kyvy7ZgifO6LRfHBR6CwdLhhx6wE9hr77BJ82Clprg5gXIRmWKZUpMnmydbrodrLcB+Fl57LoElX1+4b7r+Yg3EtG9pBV+7GJeuc9FijUwHwJ3e0+oZ2I7kCwNuexcgdd8O3YM7I3ox74CKKlVd0oklK8jzqvjzY38rX8f1yM8HTxC/8Pl9Vc8J9udlbPU4+Jsl5uxc3aDV0aZCz6DM5XbY2anqI4cRThGFxpIv3y/WLTDqGINhozAB5LlaR2IY5bw1ioSkMBrE7HSNdwfDgCSe2Z5ifQo/89OZiy4DKLdZyKGGXLCAeaD/xBJKGDB7wAu+FLBDujTcPh5KQxQ2E6i3Q+Mjs9CfQH6jwXfux/uEIPc+Sl37GrwXNG4+wbEAXEx7r1JkTisRujkyFKaCFK1BTXvNZvaxAzyVti+CSNNhZ2UPgzWwAjU6vsa5mnmKCQYQA5CE0FltMiJ3vKq/T6mX2mTPLseD32hWmOj7Pvsn1qnaovh8nmtEluM2U6PSpJk9SirURB5GwmyEj8jmekdPCqOy7cQQNdjuQW9qS9mHBSxS0BqMgcpDW1AQtx9gZ8OF88ypb+Sby/9+G5N0HbzX680z2kMSRv5tMJ45HYr2EB8FHeon/PmTQMYXeBInax9mbgbg7Eawx0+ptc+oux97lY87dp7IGrqax9qUgWYjQl7WBnbQPhirvafs6VAKFqc9GS0P1feaQ6BbWxrkpx12Bk6827ye+3KKkdyjQU1u9P0LpflkN4LxZIyaVFeWT17vz8I9a0ywIlNWkOFkRcxyhE3vvczIfOjlllQpSKC+kPtBjunVAhYa9wbuvWlPkq3rzmJNkklGCtI/hVyACqCzpLmEwgF6DsZMonQrumbaL9cOKwpRltMegvITce77wgep9gM/i7Vq0tQnuBbWMeuIJFTKr3cuNczXRm1ReXQ8KxCMjnMGNnkYaScqe+a8iI54htfoUtBnyyFnTco2vKGG77RIejVUZZn4jLu/oOqY3Gma2t/1Jtib6RU9po/ttLtPEukCSFh2x/zk7zrFOjyYBrk4XpsZxSoRgFCj4odGyiC06V35m/OcEmWOaJ+5pGWyX4f1ZHhxcPg/vp7mI3UyQBNhSJQLm6zE0T3VwbqW0fA/YsVZTUGkWORszwEF9VzGDH1qh6iYskfPzbfAKzFhu+MzAvWA77nbVWTGJIfSlsNKTYl7n3jmuhnbm57CH/UshvWfYLSAmAFCxwxw8TzLEeTu5ixkENblYffDhuyKkN/BMUVdM/e/MT7ePBU/SF36EbFw9vxfUhfh5QFpF/vYZVVI33FhCzQEZjwZ3Sm+KDSAM8EKZVzKlgNCL/gOeiiVShmZDl+lOYyNqkMV2SfyvkTTfkUY0+1YzbRtvj03CbPr+PZ79iDEH+/NZ5THRahhXB9cfKh34AbITeNieWipUX3NoPXZZzPaMdc1+uCBITGN1KlCP4qqsfE1Vd2oHUHAdCkeAxs64S6GOtQf1Nt3EcZ336YfgQRpAl4lzaWR8Eah3LV/ATkmsfoKmpZUfSmjdXrh4o6yDf7sYy3rBAf+oEJG1R0I5LwMFq7mCopw7VMkdEst3UgpXSQ/gh/mvjgUWlVscF6O2X06ZXW6M0BfINvksCmEYx+L5UNgjLqwwNXbcHlCB7IbRZCpHVxwtKPzrgoHwAKLuJFdmz4LR9t8FgjKVzyGtlgz9i/CG39InQwgSktOspSlz4YolXjQItNPvz24Ws+yaSlxsnoVMtqU5FLlA4DiSryJH5uDmXKmCG51PVjfrZBFe7Jycc9+dULvz/L+i0HRgD", + │ "page_age": "4 days ago", + │ "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + │ "type": "web_search_result", + │ "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + │ }, + │ { + │ "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDNwwmNR3sS7u8hviZhoMArTDi0jrPciqCsyuIjC3gF+3he7OPDXnafYusA0wtdtcP6V50y3aRyxKZ0x4PjtGINoXWxZN4j6RMr7YcH4qjSC+RvMT2Ic28yunEM2pJQfF+8THMI53pPETRq7kp8DkMU79aqKlTN+4PgLlhjuCCgC8RNgdaqVGCfSVuurpAmDuzsi1O1tESidSe8gIcN9BShdjDdDr5P1DTq8k8CbUnBvpviNA3pgEW7/ktApPF6pP6faiziXbND0u0mZ5IrsxLvyXHTYaLdP73S3CzFEivgh61xK4ezZ9TG6dq61Y7kzr/mqYumWOKpZ+lqdFexEfqNZ6CtEhaRTlmNDz7d5ZySi/ZTZedHAPgkrFiDD9Af2r7wFH/6VI62gJ7YhZaDKHizliNWsqaAYucoNFNoFzdXoOF0CTStQBOqNTRViusgW+3kgMi6YwC8z+vdc/xU9ol3sW/c/ayd11Qf135JOeg80XbXIQ2U1Vfx2Qu5A5cyUYinfxIFb4YDae1Dj4uSOXzakP7HC/eGEM1HN+FsaGdOmojCCpCc0BY3Ao6Ywz2YllWe0e3cju/+ycRzn4SI9Jqmq0vdbI/t7bSUTdRQz19Ddke9IkT0JhXD1ATcTow7AShna32Sts5ERpiv4zgDGHYAqLOPptJdspgR+q94tC7ePWy0FaSgovV97TGQFYJeCVuWrE9kZSBG63w5TE0NjXPVfMBHlRG89IZbHFbDgxy5J2EnJfTWCMbfqWyyds6332uJDKISgmXsOYYFey0rqSkAloHHxpOv703NMafQip6b8uXsDyWuVXVMwV+RCxXxiX11QizJ8mco5rbuSpABsIWF2aCu2O5hR8Gz5Ajw4yZMtFFavW/wBzzyrVQkkjcpSZJSK5UatvkIwhTkgo/dxtlT2qtg1UaYGL/FvW1veFlwWihtH1Xyi+mlJb+NGR8GGveDYBuJMyE2S3KxZ+g7duRkGvkqYrPq7khq3cVJKIO/gWfR84pzHZShilbaZqBrwkmGkEFvn1jxIZnOGUTnwJUuKG2bW6cAzvD9xc6QwsHsNVSKyTs6v/qiI38uFu9OU6KWpgiDddFFgfpo8ooZ24ZFaX/Pd/tkDNaUMc0lA7AvyfZjIvhx6MyPkRb9KGJteOY9JnQiDE+hljrjk6S0pDe25nSLWsyrZhZKCOn/jb0S7HjsTM2KWe4xEuRwkJ2NcvlqbA3ws/vPovmbM7xmApj5Xb9s2dZaXmHJd3H7WsB+chcrwvmNzSf7z7ThA+wzp0VRQJI/k05QACnJqHyi2dRS927erN3jkZ+2batpsKgFfUYvcr7MFex+/qkASJU+mwPykJ4Lp0+KWd4prVKTs0vnbXZISIPVBOy6HQMXHnWBWueTRle21d+xrLXZX3f8HaX6oPTDLhIh7Wn1fhjXXPs1RqBxW4IgxdGjXHFwxl0C1q9Bv1RF7sogEFVjXDffBTJEu0w/sqhMGbjFkF71uVxzuLZQpEXVFQRtcH+uw94mnhKTCanXle5hkckrpJ+ztpFh6/cUpXm+CVyLtWBavm9NiFk3ZCXth9gyU4TVAHg5OSxWNY/5yYijFNgMLNWDP44RLseTlW1tqgbQYZVz/8PjKbdYoAjxPMAlTqXEIChH8WbMnx+g5UYPVxeVPfIX5DS+2Kt7yuCH4lcKKS3I2mgxsN57NvInnD52IFb/aK1UPAHG0Em29dP0LKdchZMpzD45EPyC1YkFVQyfFliLPKddOIL8A3mPrFwXGIniLPjbcYrrLNRpHtEq3YH4KtRrRYKpRwpJRmKHz4PE7iKeOaisWXbPseKtuzLq/Vx899L2gBwzwkEFsaFhV6G41xBCf6AMcc3oFwYqXAl5jNXZScWjcsKXB6BGQgxeoUNgy8Pag5X40RT825sYEZPA2+yhBXOvyPKS9lY6PbN0BKBX87/8+eVNONPIQlFNtPEforDTk0nRhD7zDhjj4p+4MoIYwAQZ2kma9wlOEcWJ7/jRWMlowLVWYVlGrkLKCrx2Je+fa19o1ZKwHBdrxm4hbo3/eEQhFb4jbn2jA20WML6iCy6LR/BOOaCdF/4ReJmaiwexCSLxQJyeihF6he6Y9NsYJcxX/IpQt4ic0rjwvfwjvfE+U4l7i51NeNxp0pJL6ntuc7RPc0qbupvRAIyplu9e2bcGP918Zu8orsZe4r8dMJk68C5QMAzMAF34cnvFXfKy66VkzKx50tjlucXi0pq8z08XpKoC6Ceu1vxGtOGKM/lsnBReMMhsJnbeQW2FSS6gp9G5Z5v/NGBTHQPV2Ds5MGR8E0mlh3AwEthFZnyCPYzr+PKp3/onPAQUhqZxvbOonAE6ZJu5p8g4buQfPLDxfchiP4SdJKHIDWi1uPTNlBx3C0BbYbcItZJnl6w11Rpzw6Uevy48CSDWVk+3zNjFbqjihQsymYwqcc2s5zs4wrv1MAaGwiY17ouseC4VImtBGPLTjlVc6rUYwHXBVw6NCwtPFtK+Rav7Q2Su2C2SjNB8YGkvmUMdeCM4rIj7pk9WWk+iINaGYGsdpCxdxWBd2sXq2S60o+8vLHhVMT/zI1XEhO/nju8c9Lw5Xb6KfmGcRjVLnVwpmXUE19lYrIoaUv4O6xrmDggSa5Ykt3nEHdxjAKpNB6cfOKY4Yaog+PbF5nKSQmBRcgsUSs9wA1IMJ8QsBYbpbpcEPpL5J40gqtMlBIpCGRas4D6T/N21jPCQSESO64lH5Y2ynBLqpe9p+wkrJ8CtZKRHqTRLK48jboEwtmIPG6DxhVQ2eY8dtJWxaPjJX4wHEMT/EOdd0mmMUYQOoh3QX4cpsaPSPcjng+w7XOh5cfZSuaWQOD8GsuzGuUB5lVNNXidovOxvRVTeSka0RY/WkEqrkPeVAQxugmvhgVg3k7xIsjvlyjQl0UEIVfzSGg2HVZgNfFS1i/5Cw93JVsMn+yKJtG7DL//k0u0GAOVdu+CYIS/PdaVWRHADf/We4vqk7XS/RWL7pDnIrjGMIbeiuJGqDv/bAZxYBi81uq/ZkOT7oSfbNPaCo0s5K+8z64zxzNOeHtZDm55FxuzJsmHdsw0R2+dsRjjU4r+A5TPH7qNa1hJ97Iem9ZSDVO8jJdg74HR6LxKfKo2GzXv+OYII6332YUhcWtYIUSRc8D48cE5jcYtKxeg4wNAFFVLiJEfVC1aUfr3d0gIKCEhtSrSVOaMW3dfVSYC4bKKvb1KMT11fuBWEx2AgfWUSffyXaBSafM6s3KbW+M5CONXmtM4jdsYt2ogK3NDWxd6HevFl+CfKh0AkfFlITWX2GKwz/mm+BBV8CmRQUiUZNvCHN5cNn5iy3blvmqDITtAxA4hIucdDAulCAFiY0fBU8BDP4KDClH//JXz+rdezyP4K7AsgNC72tzCrRFk/GhOQ5L675SDDdAWEihGzhq+ysR6r0zq0cm4MJpYV77iD+mZgqeJDkscYVrJ3rsrJCSD904sYO86x4Rv1XbzdQ3MfPXILFaPqVMBuGDzP4bkT/sGNP5KY5O2Agutbad3QgRWGvYjJq5j7f9XJE/O/q6vdydKm+K2oaU20SGjfkRBK0WI/2TjdzMQ/wBYVid2/dUI86pZADmXNLpoE518CkqmfYGMCtzW+vWemxIiLaEkn19Jj3nTzaimKW1N7hSevPrVYqLEH0MdjqE2BRcRGJULmrWJxXHnAX2+vzcY+x8gO9SjwrdRK1YTTFoN5zR7Np2mNS1zGPAGG0/CvSjhF6B3l5zoyc/JSKY092yUeZDX74No3hCCu3+DCrY/ywc/eeJc91Gzy2d0bFVq9VDFFAz7OxlHz/TwDc2z7roSlEslWNuh8mnO2+UZhmSBjvHu03KZGxIPkwUoHtJcpOMxlAm9haBMxwYLLIzEBKFUC/5wZBklaCGgazDXcvpKVSiKhomrPkgCeeovUYnJm41cc4ha6qY2JJnG4PVlWU7+d3oju0tqRzmzwt8/7g8NL+kIZfrEb1MPwBoZLHaJnLzDEVPh/4cFPXEO8Aa/OvZXTpkijnSR8hYf6XnBU35z8IuuYQt/yk+sQVeMN+oSw0dUkpwESWpci7HenT3akVMzXqGiA/KnVDs1DXBv7DX1LI0kBh2MZg5Iqwreosq/znJru8AHtvjWEeAjviGkIko40LKiyRnf/6e82ZrBc7IcksgdsoU9bBJFx3biIeOUy/FcwQIxhAM64ygk0LHO6QglNgpJ25BEONgC2UJDMRePc9zop86OwnOfsEhMvvt16q3pIOOs5Bb5fa/vCReueXuGZe1fCX5YovyuPiLBCSlNZMBAOb01HD4qK48sv7DtIHfQ+tY622C3vn7SqxCFVgaWYWGfG3NRjMwM9IyliS+uLG6KnTtoRm0a023fJ/Q30dsJ6R+/bpcmulz5mszm4RPZVgTNkya+H5JRAfMJunWHgdkmUqlknNCMoul1d+uXLUHnW36e2cYghMqkdUIOJ3OzTu5II+CKk/80riH4LyEKOuB6UdwcXiEyAeHaUp3UWQHgfaG0FisXtuZiZoi1VqQxDzIMfA00VwobN4RgcyxZnNG9JyhM5DUifMyxASYYq5p1DXS18ZcjoGMZopLdHb8jmsvbzzRuDtqnpEw8jyQhdU9VgOu1dBXavxiK4vLBRaOBLD7Sy7QhB59acS/4CZ13re7ddUu1P+y4RD6qCw2gp24ZAceAO9ZexjYxp1dhWfwePu/9CqrgCl4hFG7hMhzMtTwzOtszVaNT7H2DuhXMsxpvv2xWAnpPC79beDMPjbwBBadhMmStqAUaZSuouVKDOJVyIepjOlxs3mpnH4vNQXGMQVrpsOYMoVOqwnRegudsX7W9SPV+FJa/W3E5In05SJrNR+BIVjajLZdEw7BPsKrlB+xRurdN8I0EitbNzIPAKkB+EeEtIGvVV7aL3K5eJGUCZwBj19EKz8WjJ8vCvcEGBqU30xZj5aSWD8vlTBoQX1ymIlFLBs6hP54JXwkyUaK1vzGuQ9YlWO456C05FXE/PN9alWtPnTYN9dUL7STxgl7aWxkD0h6ghr3qt1OCwy22PzirtK1AjvZGCCWG3BQF6lR17ZcWYi1EzxF/xM7h3wmkmZqA6oTE2uujN/Tjog5hB4yPh/JfPobQQYqvBupAH/8p8jShrQl1JQTmQ6VSj6tUW7rDBpYtqmVyq46jtvK68ftD6+jC+8gCiCh5VhGFnVfMvwSsd7P/514k+LYGk5pOU3n4k1hXpmx3Ezi8Fm5oDLOu8orA34zOpoBT+vN6hOBrzyg8EowVLfZvSCqR5qYWpAmuhCRb/eKbKSzF07omXWpcQns2F0auiy9+ObhdjYgNd1FPTL0blJaOGNDnmLdnqCoxGdz75wVYqiIZbiX7+2UuEI82nw9gig/7taQRfKiU2GteETl0EAwNYsMsdNzfB+ImiFbRA8OiuslXYbbMgiSTHQvzGPbE5ysvhbkB+Y9HBdwGCuNryv/PwGluKI/xeBeJ3Hskqu9EnB5ZAoX1nI7CfygBf8XVKFMOXaU9NwtHEV69hwGykTqXVAapHlE/+0aQF+Y1cT+7pnNXwXjatVi9ADoKXTOmWDsWw8y7RgD", + │ "page_age": null, + │ "title": "AI SDK 5 - Vercel", + │ "type": "web_search_result", + │ "url": "https://vercel.com/blog/ai-sdk-5" + │ }, + │ { + │ "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCxiOiFekHC0+MvX7RoMvlOqtxzKqzcIRfyoIjDhmxPG7soqbnXgcZHvSGeQz0df3ps4/KyKeXwAskO+SwblhuRtkZAsJhvxY1Qz1HEqyg1RCXfzy3sED627heCMyoK4Q3PVRcVHwJktJnn8iN8kmR9BRwKrsQ7lSK7D3GHqsYCbCeBSD+2mSNgPp/f5NpykextjwP1G1I5kLG9QafxUBns1mw9ySbO6h8l9FymB58c7Ba6LyirXzUiMeDQ7AmnNz71QAj2w27lZjDrWRfIXrMumCGR131fR8zxgc2uifKVD54oHwfj3NsSZUEHCaoza8hEHnHlcptNPymIVq3bdM2Zs8Ko8fICX5prAT9gIO0ijH4S4Y62nc9yDf7qvEa21jv1QzijYOyl1DIxZoKmYAhAEhYCeMsJNqenZaC9vdkCg4fHxJZMeI3XMc+iUX8mUBqy+YkY5XxIV/IOZS7Cs59JHpkn0xSehCcPHx9aZT1jaKR6UyHyB0hu1OO6rBE9uNPX6UdsA4Z9hF9QuEKGzqqiglEIY+FpWny4PAFstSTm+UGeysoCwxVA930oUmBA6HrzRbT2mlh0vZ7fmWAoVvJhofvPxFBKEmS98g6dqRARUbxXD8I/PK57t9JvolNwRTTpLizFTlu3wc5bbhX4MHYF3kq2shYrbgswyQLnqRoA37AGeb1GTDcB+vgG2S00g96Qn+X4ixNaw1maFJyAONo8BX7ovpqONNLv6NKZw91M68ERoTA4/enJ+SSrEwtuVYHoff7+SaDDtf17GZter9g0PdFAhwHrpnJ4ccMtVjXlnOw+Nem/muIYRpVBr3WvYFCIjt9W05xVrMfFUwkhAyZQWPhOTj0W8bWc/q7sG0o+uL8Js2oRAiTB8fYGxV/YiiIO9BUlbPYgxvrKXwmXsBb2m4PQTyY+9LpiG7LvjV4dqRXtBomgATl1BucRViBp47ZhCB3YAJBPQmO6GbKt4kJtBZoQtWEq3U0oZ26pwLTd9EQpYUzFii8Igq22jxO6Tx6cSdRD47gx6FbUpgQmMQJekjmf2mTwZB/u8umQqNnpb89tqsIe1pKyln4OghmYVVn7Mu1IL3AlUq7DZowA2Z3spUY0O3gRvMqqyLKE3Qg+uLXaA0TpHqnzcZ0RDHCW4yPTvlhGedtKJmQV4WH2m0BUSErPM73tDVzBw4s/z44IsNaPIkfVEy/QOYxkc8N1Ifctzrl+y/r/o/lxahI/sftesVYkdd13oh2vy3d2RpfBGXQWPuGmHRMhPy2jtTVSOoRvvffT+A+Z02u8D34/Q8GjLWB5XJA+ktdN/9EcmXLf4WjAqTNkJxBPScX53HFKURqev9Yx9hd6OW6Gvtyvkzwrhkm9Yir0UmAp8Z64bG6X1hByWxggnzb/eGlN528C/zcAXIiz9W4kXmycJnaXWt2QXSZbW4ONQJTy0dstA/O+rWKEB+/u5b2Yh+o57RUZzUGW9M8KOSoerX0agWX9vUnOzL4ZlAv3AgM/i75gL3V/+Wc2L5pfXJRRzL2b9b0axLXPZ7GN6IJU6KhV0Y/s85cEnv0jfUVKOSQu2EuFlz577MR7A6D+Gp7H9KIAoRl6z49BTi/Wv3FWqYncXUqkXb5OGMUQ1Ro20A3fchb4Kiz1E1W96YuntEfyfBpsFFAGdrDlJnhLWOuBYmzCmvjG892f7S1sIrzkoxZVZM0kWE7BPFNwGUvnqqx1dki60Spk1+zNUSoNL99QdeFxbMydpgucX9RuP4YqKc3mUPvmCfiLaTBwrW7sHOxRwizbIR6ge1MQ3BLzka//94LG20yy02cZYkjPGef6tlewlWSRhAV7ujhXZkJeiClGFhBzc6IzauGsVTktEC+inesqoM3ytl26GHTyPp7bO/9vpnJ1LwiAjyccpKuzzehPzdTA9O/Id8tZT/QszeX7MpQQwTqjaOpE+gHFXvaJ6q1ypHS4vzdFJb+2b3VN3ZF1PHdFm5ql0nvnetcgOhE3hUeix5NGWmNaLkTSVS2GucNPP+7n5D5qfy1hmoQct75xiEpuaV0LnjHO94DIxyPtrBDVEgwe3pwzlAC6NBhH6n8zrCPV0vyZsU9/pHL/c1ItTTwhdA26FcN4MGyYpHTHEYHBEtNx3iNmdvUS9GeR2d+iXZ0hAJAsmp8GpQR/7XZmJ91fRT+NfDeE4Kidd3gpwL1qQf5mswB6fj57ZuDDvuqPu+EcvPyTAIG44aRvTyHufANznLGq3QGeKf10QeN3Pe899X8AXaq+MVhKTdRnNzwtnR63Xs4NOln/spOod04LOlqdzCC/qPrQ+GtHTvMH6/0APbHdGq94O8NwT2JJmMfF62hQh3cD9D0LE7Svj/4TmR+wQfVaUnuH3LAz0MszvO8B4ypZ2lD0vby9JzGaKZdjSyvp+HTTGNWZMwp2Dt1doGAM=", + │ "page_age": "3 weeks ago", + │ "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + │ "type": "web_search_result", + │ "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + │ }, + │ { + │ "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJ7EUXY9ZeIg68y4JxoMull2xOlYWmKoYcIIIjC3l2xcRJmgibQjrVVH0/EEySG1q4WZW5lZ2OJIEdGykgwR8RQnyKICnR0fga8cpsYqtRB/d4KgjPxAXSoVzQ3nAChKBZ2C/tnl5JKqB3vx9JR/bP2nQAKEg3IXCE9nDgfrYnilLA3k5EN27b46aFQ25GDCCCFnZtrCJzOo36YwGYhLGt3nKQFIzZwjDCFMwwRgom+LPIA492icli0qBx+Np1H+CURvmE78X15QRokJ1i7XkmygmnJ54K4TpFUOlsuihvy09Q4HOFtCuH4z71tIJ8fADvZK7fhNuThoe5FUXX1Jl6YK4PdvSfQHbrkyPJWRwONY2TuYDmcR0lT1wWVxFqdaunpSAl5zs8YBiQEvBzdmtZWVQFIEnPLwwBiT5vS5fl+1DEDVQehfY/eK+yjkZRAzyeUKTkam6k34x1KhijeqRWV2GDBmlpxMffxbMe1kw/TKPhjV3aHTWBMufRV2NWntdUPQB6SvUeV5ZNksI8PeabyLU5KN4SPgxid8JpVCgx02+0hxXzQin3ywdZ177PoZ1Y3ZFct10hRm8XDIlEteHddw0fH5ogDMglf5RBImlmmKp844a7qgTDbmXL58dlb60JtnTN+3A4SYic49C6HdRhhQFFqfeVSVd5BQ9hFuj58gzGFJkDFY2+GEPof+ZrsQRELLjBeLJEwlXMvCtyJFmM65y0I3uMR5KtL+6VzICvKTb1T1d3Dj2zkuIoLqrVlGLK2vrMCL43w5B5xL1gIBtr7hnrntPZ1R0tKdiyCUTjoODIsnfsTHZJIgNIQRHhRBqGPO1IfAhZhq+ofvw+CeBuVwACwQlhRu5vwqU5MFUxhrZcNuzPSD8UTYAu2T6OFlhlSXZkUhYIarZl14GCpMnfX5j3kHtjdwwbxx+VZB8Acq5D41iBpqvlLiHK4Yb3uW4P0DtNwKtebWwwNWoRDJCNUixaCHzjndplqExlH3F1b05NuOlq9MyyXyp+NgZ2dIPKImpZD46GXr0mQc+NnXKXmy+brdFlLdogd1FaVJ2TqCYTRHeKwSwlRHsah/WAEwohsNaew9KPcJbSgLMdfkcJFKeK5z8VfrLcI5jHeeI+maV4h2/n47ifjXQgM3wB89Jxw1B5d/owhpRdTNZW/5BdZqee2pj1bcvw+vk/8fdHy8odkdV70PcKyB7hdkX/53YAdn0sU0r1SClsbduqDXgR7HxLzYmxXpbJd+LpM5hvFDM1sOWF9oR9dJQHNCCthQ498kqSinkShw1pQxLLHtgB1kIYNgv1CTiGg658JVfmM2j2Vd83FBzMhXf9oMA6frXcDuQSCgaBThQoU/CtxLJzhXwZKPxBn+7JoZrqZFuf7q+zAn4k9pCVikqInIlqyazEfmZY94URwvUIE+yxP+ZJK20rCQyKheV+12n4BCscteGkX4pFftk9zjkYyuGnz8GwTApxvJovtodp6TutDPtzOELD9a20873J8HB1SVlNFt39uFdIu1lyVqOZA1fgwW9g/isuh2P/plaUkf6voz1QR/xeQEZgpID4cg+sDybZVnzEdnx+3NVPlBeuw9kts4nug4xcCD8cBobEfdkdOp+XrkiyLrq3Q5eCzCUydJ/cYxGcQSWyiiP4oJG7ty8IbEZKzi33gChlsVWJn0JqgH0o25nqk7QHOJlM1oeQWMb+QQxw8nXLJlzSKiCv/3YnyT19gtDp910b5CSyiIL+teZPiPG8twslWyOCoKilxLUt3S7tD+V3PBO+jHn93KWd+fC5NSZ62kXAcIYFQ8qAC6jprwOiQM0GupcIqAz/bs1G5LQZpNmaIrAsGksmpXBMJtZsQCUSSIAO+p/gq1BaXfefLz0aR/PP0f4+MsKCpSu8IA+h5e93Xlb0tkdWSqi7GvPM1/AEaTJqKQJpPdOhdvIQAKqxFs+g4iO/S419cLTMnYG8fJn2ODZVkRem/8oYcMEHYLuQkBD3LLWTDzxWLzdnwfAqsRl4e5/eXYBf2W2sUxwkAI9Gg7B1QsQMdsxLRGiDFv9puttSyLPnVaOVQ/SivS8IFJkjJS1nQOikgWq0LapKp1WynbSHI9ZF3AcUtTj733+oDx7vJArxl9D7L1WBpHn/6hbseAV23Rn2vK5TSNvw01J7smrlG7x1ZZJBwsse7qKKfM/lTtiGCW0HuCzMcatC+iMLAXAQbJI+Lu1GP/J5G3KdtDC7cSVpoJv/wGYsU8R4Vf3v7R24422u/iKMOrivetZe/eB8ubxoWcsky61n7Kf6/QZ5gwUPBBFoMSOlpgQsdbexlMeUECbcRj28LeUS3XhSs6o+w+LPP5VCrGBxOo9XzEupHwUDfhSbdqtGyJGmg3O5M0PdoffvFDyFlPu2IiNrNj3Ke48k9rJKWdwuGHryvSLXiAJKM+iYiWG+trste6cuzCjtV+KO4GCWxp6Ow9kbbDbuDXFPDw7KAAzKuVlG3fScbp28JS2zyfKym42vwXbgVNEA4p6LD1HN0NNRv6oCmUu852j37MkD7rIfBEawSqli5P+Vu815CJNSYRwkfJwIXw3Dc7lbHqoi5cqnzyCNIkjo5rPl9swjWgS03WdXRu2pts4oHkSu5skCwXES6cIvCqebozVyMAlYIFXvqPnSyKvnEJUXc90fuyM6NskP4UiknKT+sLTQc/KHGYl6QrdgqenLjzjRbBr4ww2PdFbPeujcp/RlecxGP8aeBe3P8re2jqTp1yQl1CP9Tl35fO/VbdahyW9xFETwRg+6/DAbe0ZcxeCe4BQxzBJYh7t3IlzcdcDd3sDGFgiKMSunAQ4JKCG3dciHZgFUzVaVRXxnW6P5cx66Em7sTRR8J0g2t0ypFQeSPHqndzPgESousYAgqRp1B1YI9NW6jVDpJpGAM=", + │ "page_age": "1 week ago", + │ "title": "ai - npm", + │ "type": "web_search_result", + │ "url": "https://www.npmjs.com/package/ai" + │ }, + │ { + │ "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDI5QuK6Ul2/hPgEs9RoMZvr6kOyVbnrig26SIjCKbzvRMUF6XUs02cYL0BpmSRn5lwuA6HhHvR9iXIHkHpZqyP3i4Q99tpqK+ECmeE4qixd+BsJgZw4q6ZnS6AHyw9pEnrc1ANbd1veujyHZK6qLSWp5KWF06ZuxCVf9Lj++Lmf+hOv58DJFDGkPMWAuyPquvUaAcUwqOFSjJWwbRDXe2gKdpK2up/ZWKefqk3esUs1+3RRsqIwshVc8urVYkkavtO/iUfQPR48h8jS/nPoSOEVHmxGVtpncVxfaT/NwnRxuG8Z5J1yviMTQcVGhkVWprBi5uwVFhJMKXPZsRDrM1fVDEQYmLpjEbcqIt4P8nDwGzopRR4DbZg3TmuDQx2kYw4mcRL9IxCY/IKYDTru3q1GMvU5vBx+b3LEcwAZ2eenIo9R7i5xalBFcH7taf6NSeIgRHQiJibXxNqS6RoFWFoi3Wg95WoleArqiyBklzs9fRLd9GTIPOi8HlA5GadUZBjJ5Ds2RhHQ9Cuev662cFJIlWtdmSC7m47Du0eptwVivpBYEi7g2GmLGWT9xRbiG6yHbeytEsyfRmrWns2jCNvvEetMkWBfcaNXX4fAmK66wtrylYt612QRmDKaIKz8h3ih1WgDYD322hRk9ythJ3Csjl6fkZl/e2pPrGd67MQ/6fcx9dU1TBweOyrcGJsAwCgLuetcj+6RH6XfihaNLWiAr/GFMPVpMmWgv4PbQ2Uknc8UnBKeIpLtJ/Bt8chdU/3PyPpAV77FGiYA0oiB69Gtf5b/wQaAd6GHl42C4bxi4TXKyLoPhEUIIJXX9WR/J0cB++5vuGtc+6fVgaOFJ/TVAlSOS693jTnLKDvrXhwkQFoF3F+j+Hx/m0C45AGBSfjXjvAU478opr48uDG/+FIMxrkvahh65YrWsKwlfY8c2wSDBx0sDGfjjPjJN68sKeWUi/RRfvgZZxcTOw3QWhRaDFTKLGyKPID0GzEkAQotfeMcQBuQtHmtd3nOXhkpcHsMxZEAjqImnno7U8G5P4EJP4DRxG3CZsymlmjrSM7BTJa/hNJhgWX5RQhv/N2OUNjP1mFJxG9UypG9XBbJzCm5q0p1fWB3lWTDSMYf0rgZQFlmk+MWYvi4sjb2i/6RwC8BEi4tpRS4aDynnuCgy2DgueFR11+90fu4U3cVLcof3lWgzyUDo70QEMEO7BFQgtt8DUXCNL9N6PhYc5cFjbpWspzGzK6dvbZP87Rq7787LQ9s9oRGGvmcctR+GcrHDsQB5YZhjJewbrJAw9VcL9YOwuRX5ASU9lA5EFewd+JV6bQ+R5DICC38YHcSMOakJdKLXE6x3WX/3aT5cnyxxUMAzJFFlzBVJ4LDKGbCpQrTrlfUVH6DGdSvYgcN46yqouA5rmQr3PmBXPkuUoyWcGXGYDs9IPScNU2gixxYo7uQBwAi8pHigxNUL+sePovti4B2T8YNITTxOYiTjlR+J+mklkpfMY8GgHi6tpzWSyIqwo+mo3whkdR6YdORLRJHCX+DxzpxYpALYlUnvhorCBoqdeK3jpkL91NQQoifX76fn37+6xHDfywmbQslCW04HHmUMKDNgM+xreBavnWJqDHQQuwxgWWvyZS7mtRa5m2P8UjOFKo5RZ9uB9V3bwdajdeHKShfOapysQTOpAhvptk6Pm52IW0yUsyLKKQaHhJv9eRDC/xpviTTiCRCf+EbZL+h2SCF7IcBYeOg3dLP/FFMchaLfHVEw0Yz0MwC4oHPYI5RDfgth+M5wan7qt/l7RBEYRzA3yPh2ukT0rRPjqbF46PjPoZg53vTpOH9WV+0xEIRrGFiicp0CtG6AoVIQZ0B1174rEm2MddaLfkqLHvO2DxCez8raIiZcK073zkqbrZp84XWyzfBIKCSSIyVPRs5toXWgofhVsiVIuZUn25gnUt3Gxq8WvovPnL1rsMd1ZoPlrA1sXgnxPELCzCSMo5S5BcdOoGJO0zPuWPL0hjhOxxodEp6vqy6VcB3APbgTW/Su2knS6uTI34fQC1tpTvTwJ37NPj1kjWVLFelx+NvCvWg1iVXTkoUtcFCFTEbqihtIADigaUPHz20bLeaqkIcQYocJDNfdQLuNeKf6EdbWfvDO00BcQ7g/dD3veGN2rTdw4TO1cID/WjQ29WqutFmy1I4WM55OZLaqSNNDD6/k2TSsZfzCUnw22iVJ3eTpT4Lwzqn4u3UX6cltjb+1R5iizU6y/UxAd23vIaev6MCDfe3p5qiuI3JAbQCAuRDAYwaD9Qqa8FHzdOcPvCmTkIEZ8C4zzcK3ckcHTmZfs19MDdvGeB7Xxa3Cvv7q6dp1o0sQToB7SzPRKi7cChvVYEPq6OATIjeV9fx53c6FqA7QuwbyMY0hjdPAdadHdYZyKlT/ILySx012VP7CWy5bCLEMOmhVSv0NQ/9yHc92YmuUZLfiUIKC7i9V2i2nQXW4dkqZ4tVJJmoNUsu95H7YfLwetGeO9nyzL19NTSoEv0XQOfu52hzUSQeZsYZRuOYg7o6LzE4rt+FyUkTKV94HRFOTRm/HH0EokjKzNl9MC9O0MLtXKVND7MM1ksW48JhRb06Yo9kVTHNHNZT6CR9hYWgU7sNqsdPsQtecRp3oHTBi0UuSFxcN5aK/Iv/OZuDT7YVwhzVEKxn+C87qBRHn00ASNXlHUFjZlt1+XOQFYS1lLgzQ6l46poBbGT8lL6EvRV3YZkLF9P7cx5lIeDN9VDSwys1SxzOhZpVce/Cpc6vvq0S92hq0imLtxnsEsh36D/9QrHugf16WJIo1I0XjcbwMxOWe6VtXhZejmJ6zPW3LhvPtF006/+UerGjmE7zqO97jkkRD3AWuOBOUowzskdTVzPj1D3CMx8WGaC2M+V1DgsGL5KCsva0n5YJANMFYtsZKYIIJEeFedi7onBnjw7wnMAHD1PkX9iah4r3sy0iwDnj+kJF/C5VKy6aYGQXJ30HBTpka4gUmqM2tyrpH2nh5R8UxdAfyKhfAg3U89ZIa+e+9XTkv1UV9ZbIFi7UuzAU0p+wmv0qEDHPTqgUgB9z8uGZda0xt+n2MN1n9N6cMb6GDhpC1QyG7MQSgbud4oIunQ4DgiF6bmvoHNer/dLEzBZrxJShi/PRwrtDzyYm+nJT1EvVqBeuhwtc/c5kGYywn16iL7bA5DmhGl2jG/DKNgTuRfcqVmK56lJIO7ISkttKi1IJcvlo1d8nKu+22BZufRikLmVh7Kd6nMS+o1cd9ssfZGipod5E35AkRLIll4wpc0zPb3GnSAuZyTw26lRx0XwMoZCGfVJyS28EkmrhAC4FHNSTkMD4ERsFcIqKp7C/3TOvEtjMRoPH7cDeBOgyj9YbZJD6y6zOFK3YDkhOgoN/KcRX4vKpGv85kF8HIMniRhL7hb7GfR6ASvrHnYXQdP6otUcfzGycG4G3K51NV9qgr7Z9klF9+Wq1ZYTyPoShUd5hxddJ621g/i7Lm2HCd+FI2Bfo3XSwi+DqyzjnDJvs9M6K9f38HTjlMumBz6I++Cun/fwG+0F5XvqFgu+F0kov5+Un0Nrelj+FZ1fewK4e7ApjamY5Kkt5jp8gHOglx101TUM3CRVrkHurclOkTzF3h2Nx+rjHUEAslSYhBuOA4GiyBNyhEvQGl+vQYr2PgO3ERREufR0X1IBWDr7aTC2xZrqKBODB8EPkiyxdvmcrMvc/tv0IR4NjhYfD/LLD9tdqQRvJ30kpN5s7qQTMhgP/DzDKGfxa8bHB5qOlLq4ERRBE/qDvJVlNkJpSXUMA27jVCQsIsGhHw/RNtsXjCl6fvl3Bxpu3emibMOHm4deUrjtWUkxpXsgYbocBQvuosr19X08gm/A353UhUhGF87kFNeEJ+b9EOm6M6zbXNdlIfmrU6KG7qNen8hUfYE0YkfCjZJBRLC1Hs6aCrczVELTIDdO3Z904lr0Mep8vd9cvWthvbhQzr/Kz7c2JzHoNIFdHVoiiHHvqllkmSa/uegGw3Eecmgnvx/YSua1O5LFhhJtBDpcYYAw==", + │ "page_age": "1 week ago", + │ "title": "Google Cloud release notes | Google Cloud Documentation", + │ "type": "web_search_result", + │ "url": "https://docs.cloud.google.com/release-notes" + │ } + │ ], + │ "tool_use_id": "srvtoolu_01EzUEcqxFau5oLsH3dgLhiW", + │ "type": "web_search_tool_result" + │ }, + │ { + │ "citations": [ + │ { + │ "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + │ "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAC0dqze+RTbPFWgcBoMh1hKOmIsQigZKu2SIjBkmz18F4Lz6rZMcRDEOH+QSMJzaHRsGNmUYqrNcYC/WbzhT09ybQeC/utv0msOSnQqE1shyPW+S7Psl8Jy2TvVNnzLikAYBA==", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result_location", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ } + │ ], + │ "text": "Vercel AI SDK released a patch update on May 13, 2026 with refreshed dependencies.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 256, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tool_choice": { + │ "name": "web_search", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "max_uses": 1, + │ "name": "web_search", + │ "type": "web_search_20250305" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 80, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16216, + │ "server_tool_use_web_fetch_requests": 0, + │ "server_tool_use_web_search_requests": 1, + │ "time_to_first_token": 0, + │ "tokens": 16296 + │ } + ├── anthropic-stream-thinking-operation + │ metadata: { + │ "operation": "stream-thinking", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with the number only.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "signature": "", + │ "thinking": "The user is asking for the result of 2+2, which is 4. They want only the number as a reply.", + │ "type": "thinking" + │ }, + │ { + │ "text": "4", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 2048, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 1, + │ "thinking": { + │ "budget_tokens": 1024, + │ "type": "enabled" + │ } + │ } + │ metrics: { + │ "completion_tokens": 39, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 49, + │ "time_to_first_token": 0, + │ "tokens": 88 + │ } + ├── anthropic-beta-create-operation + │ metadata: { + │ "operation": "beta-create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BETA.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "BETA", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 + │ } + ├── anthropic-beta-stream-operation + │ metadata: { + │ "operation": "beta-stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + └── anthropic-beta-tool-runner-operation + metadata: { + "operation": "beta-tool-runner", + "testRunId": "" + } + └── anthropic.beta.messages.toolRunner [task] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + } + metrics: { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 607, + │ "time_to_first_token": 0, + │ "tokens": 663 + │ } + ├── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: "The weather in Paris, France is 18C and sunny." + │ metadata: { + │ "gen_ai.tool.name": "get_weather", + │ "provider": "anthropic" + │ } + │ └── get_weather.lookup + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01PWZpYgBWx2yCEBqiFwwtRW", + "type": "tool_result" + } + ], + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + } + metrics: { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json deleted file mode 100644 index 7b7fd46d3..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json +++ /dev/null @@ -1,605 +0,0 @@ -[ - { - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 4, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "start": 0, - "time_to_first_token": 0, - "tokens": 16 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "OK", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-attachment-operation", - "type": null - }, - { - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-with-response-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "stream-tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 26, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 713 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-tool-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 589, - "start": 0, - "time_to_first_token": 0, - "tokens": 645 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-thinking" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-stream-thinking-operation", - "type": null - }, - { - "input": [ - { - "content": "What is 2+2? Reply with the number only.", - "role": "user" - } - ], - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 0, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 49, - "start": 0, - "time_to_first_token": 0, - "tokens": 0 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "thinking": "", - "type": "thinking" - }, - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-create" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-create-operation", - "type": null - }, - { - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 5, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "start": 0, - "time_to_first_token": 0, - "tokens": 18 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "BETA", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "metadata": { - "operation": "beta-stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-stream-operation", - "type": null - }, - { - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 15, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "start": 0, - "time_to_first_token": 0, - "tokens": 39 - }, - "name": "anthropic.messages.create", - "output": "1 one\n2 two\n3 three", - "type": "llm" - }, - { - "metadata": { - "operation": "beta-tool-runner" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "anthropic-beta-tool-runner-operation", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 72, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1294, - "start": 0, - "time_to_first_token": 0, - "tokens": 1366 - }, - "name": "anthropic.beta.messages.toolRunner", - "output": { - "content": [ - { - "text": "The weather in Paris, France is 18C and sunny.", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "provider": "anthropic" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": "The weather in Paris, France is 18C and sunny.", - "type": "tool" - }, - { - "metadata": null, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "get_weather.lookup", - "type": null - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "tool_use", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 56, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 607, - "start": 0, - "time_to_first_token": 0, - "tokens": 663 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "type": "llm" - }, - { - "input": [ - { - "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", - "role": "user" - }, - { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "The weather in Paris, France is 18C and sunny.", - "tool_use_id": "", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic", - "stop_reason": "end_turn", - "stop_sequence": null - }, - "metrics": { - "completion_tokens": 16, - "end": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 687, - "start": 0, - "time_to_first_token": 0, - "tokens": 703 - }, - "name": "anthropic.messages.create", - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json deleted file mode 100644 index 0acff38b6..000000000 --- a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json +++ /dev/null @@ -1,471 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "anthropic-instrumentation" - }, - "metric_keys": [], - "name": "anthropic-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "create" - }, - "metric_keys": [], - "name": "anthropic-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "anthropic-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "anthropic-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metric_keys": [], - "name": "anthropic-stream-with-response-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-tool" - }, - "metric_keys": [], - "name": "anthropic-stream-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "anthropic-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-thinking" - }, - "metric_keys": [], - "name": "anthropic-stream-thinking-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-create" - }, - "metric_keys": [], - "name": "anthropic-beta-create-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-stream" - }, - "metric_keys": [], - "name": "anthropic-beta-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "beta-tool-runner" - }, - "metric_keys": [], - "name": "anthropic-beta-tool-runner-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "anthropic_tool_runner_iterations": 2, - "model": "claude-haiku-4-5", - "operation": "toolRunner", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.beta.messages.toolRunner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "provider": "anthropic" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "get_weather.lookup", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5", - "provider": "anthropic" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cache_creation_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json new file mode 100644 index 000000000..a46531cc5 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.json @@ -0,0 +1,896 @@ +{ + "span_tree": [ + { + "name": "anthropic-instrumentation-root", + "type": "task", + "children": [ + { + "name": "anthropic-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "create", + "testRunId": "" + } + }, + { + "name": "anthropic-create-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly WITH_RESPONSE.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "WITH_RESPONSE", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 7, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 15, + "time_to_first_token": 0, + "tokens": 22 + } + } + ], + "metadata": { + "operation": "create-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-attachment-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "time_to_first_token": 0, + "tokens": 1415 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-with-response-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": true, + "temperature": 0, + "tool_choice": { + "disable_parallel_tool_use": true, + "name": "get_weather", + "type": "tool" + }, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 26, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 713 + } + } + ], + "metadata": { + "operation": "stream-tool", + "testRunId": "" + } + }, + { + "name": "anthropic-tool-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 589, + "time_to_first_token": 0, + "tokens": 645 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "anthropic-server-tool-use-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + "role": "user" + } + ], + "output": { + "content": [ + { + "id": "", + "input": { + "query": "AI SDK release 2026" + }, + "name": "web_search", + "type": "server_tool_use" + }, + { + "caller": { + "type": "direct" + }, + "content": [ + { + "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJbsVr2UHzFeJ5hhpxoM3w9/4LAO3KSyZR0FIjA6GYJG7GThp7LNkiRYQItRnLnS1wy6GU3paY1h/NMJMX3SNbIR7AMtYv2IBfIaQ1sqzgjs4Hv+ZZwkFtM3T1g97VKd7F2mpPYgV3PtHB8SkrDpFbWmLFqHz5qVfJJQUJnOhTFfv6t3OGg89s5rQQonA90YyteQzaj7QV3EsTGAJKgNRbjtNby3IaKtBzH38NfoVFBCqeXWZy30qgqPljP2otYCNtOlCVvC0yi4DcsdZqoBMmvaozMiCKJVjYHzCbBtzIM0/7+Ro3hZc3VQQ67NqcTXnzHMdS+SEcNpoUu8od4psQwxXKjfUofgHoVsDlSft8FIOA1T4WlJBipoQgX2vfU02AK+cZ5ZdBAfVRYqewaqYVAgRTl5ylaXG8Kpwl+wFm2ZkEUbVentoJWJsAqFxW8xHFBAvl5obxVta182X6O4nocHWLnl5O+d6vgSL+ayc4hp8qiqPyVpPpmPFKtXeaopQGEfvp71LKNNFjOm3Jx2w+bPo6cbb446GnZgEook7XtJXp7Jty2nJV4zQeWbIvlwlT2BkPxmYGuTGhNFcvs4DuxRAVGOxsTEAk5zhNr9+9uqPPGKKmIwSkFF4jmGE8Va5otArw7nVT9Hu/mF148pkP1Q2JndOlZ8CaKMG8j853+jb6Oux/VocGVpJRdEfR7prBxh4A+ubi55MmSk7kGVNAwXVfCe1PSp9+p4iHuLzVz7h55QWqWLlWUS2UJectPiyPAa1QMonmFDi6pWv9ZBv2Iu+6yKX584oGxwWQN/fipp3ZYPltgOKPBsCFJXKmo6T+Q601uBG9cMIGzRgPl0pibytQ+ag91vjw2bgn9ASAeGE/MPla8BYsXqaYbk7lXyRffmA0cAChdRrAlRh7pgc8vlKUVLR0+hB+3znNm4/3g/RRiiQlNt4d4PeEbA9iEP2SSpZI5tYliJHbqEAhuU5wskpP3JIrTpi4skvIFZiSeM8CPF0HwbYJJx+kPgzKowyV9eFK5OfGwrVP1d/Br7N+5OxcYtqiRRxrBhnHZe/CrzAfr7lfAW945kduFMmb7s4UzW8xQly5gHh/AkoH/7vvogLCuiMGO3RH8awW3aBrSWEWWJm0iQXmTJBCT23zhHFRqB09PiYIh7JGHoFpHR6vkf09WybzwvULpFWzeeiqFgkLvpqPfhsX0DLAXCpgoNOgQ8woWxZ7m2B73uXt80t0RrlVCribOlM3c9PYoJWQvUAgQ1VHNJRyX+j3zsvSufzzpk1WOpuYf+lap2vbibDtRZ0h72Um0PQU8S3bHmcWRz1bhE+B+BLpVittnHblDr7B6ClCfaIZZvVPxgVvYhyCNsoaSro2Dqdb9UR48UIbp/wBJrsKMiFXdky+jLoUzdt2VqJ/dRXxQnPJnvjnDKrsuzaOxRXnoBcroQ0Z3n7h3JxiUJ8MEj9RzpiYfxocPzL3Ao71P02AgpExzTHkIgXGrSAGCpqO2jvst6C1hRe6kJWhQMl+tnJqmIFcmbOlu+WgmDfBGx9ZKr8DHlS+mgRXb7FORsf070qWQHsKy9GAM=", + "page_age": null, + "title": "Releases · vercel/ai", + "type": "web_search_result", + "url": "https://github.com/vercel/ai/releases" + }, + { + "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA2xoeQuNMkMTpYeFRoMn88FQvPps0GZ08VTIjAYn9VT5PJmc10NKT+kxwg7Y1NjAFzgkGnoz7Htd8IFlIySTKcGf+xMlPkmY+uUpiMqkRmCopQezkS54y4krvqDbuYq4j/lvKkz/OD7MYRFf3NaCXIUkc351WWWHg2pN04zLOiH2nAy2Hb0fFH8XnWngUr2kN2/nAnAEOEzJ3JqIfmf5yTgghjoEo1sAaQd4BlRa4vlmWiDs4q848v/6kjluTKrC+RG4vt9uPrCJDdqh11AMoZLi6eNbMMfJyQW+RUOSMsPSIUpPWKQKEFQeVkq7CbgNR7jeNHK4pNA8md0DV54TKrEzkCEdzTmvVPqdW3TZF+7C8mNKBfae4yy4uFFYAzyVIuv8+qriXgsfRGA5rnETQeqDZT/bIakCNX7zd1eIn6c0q1ffymi33VsRwPklPlK5I71HrfCZLXUKeeMLT7tpzDYL2n9jUeXgf9JWt8cOeW0YdRNnm+a1WdmfxV1JEoYHG+cCQ1jsgmz3S7ZxqS/ZekjeqM31WAdqvLf7xDasMLKq7mNSdjuB4ILEfG/RfOaU5WxkoA4YzgIteaqyNFIUPd91HXA8y9nWqNBE0o4W8Tf/rndxO5T8vferW6oERcnSU51SHKVVDoMjcCXbs1zuE6iyUIDJdl3vLGVL1gnT21V1yq1JkvAHaWOmxbfGXfVIAhPdXLocN7ZqWH7vXubGDy8FeomRLNyAKyHSn7FApgrYjH8avA6blhrK49CFc/d25IwGRNiMGmd0gD9aKLAocdwa8LPusRe1DIl4349eUYOTUHpwPHDTNNoZ/E+pZHNnlhVcsPH0IHewFLo11dmpCaLa/PQTr5AUyn0JRbyyImKhJEgxRHxMKs8lqQ3JuVd0lnoCx2ue1EzkRP+JqntseNo6H1zpXZ6W92J2yF1/iSbpOVmeKsmNgm5uC1c8YAU/HgkAnFG5YrFPYLdYKp426Gb2r25sKQGpi0XckcRxpqmUfPb4BU0YYGLCOQGNd0fr8jlK+rL+gwyanuTxBLlpCnPmCk2zy4yQ1kf+a/WjVKt7yxAc+3d+fDd/ijIPK9EuSjat+5G3/pG3ThUs7vYsbsXdAOcxrKO8WSJL4xBp39B/u7V7aGYmZFrzypEUUFWif391Bj8gGSLGTYu+qubIusI7ON4+LVO96zwUJdnmzwJxBb5sjCXWlz1dErNSfbqd0PMYoif6nT2z1VcnGCDgpT4QNGjpD+sS0X9fK5qa80mFwB81jwABGOuki1ZtRh4Bns66jwPAqLocbw7+5A3+OwKSYtolbKtg0V4CYOYKb1W3Ad7lOkFOJTKlkp2zY/2Sq/VWsGqGe0VkYWLI6AutQVtkVQYTizXasgMyvrND4Jl1Q5rXCYoM+2laUT8JFEDqyU2tPlQUlzVtAX7nrRawsUmzFAnic6dsoZDRvAHBx0EZClA3+Osz245Va9V9JMcgEJLIqWOmJBSljQOdjLEpFBimtipHeAq09W7Eh/srEA6XhWXKhCoGZuQVw2z/DJ9T2f3B30aOGczARlYu75AM0YdWvDHqzSza3oCnZkcIIB3svkcc1UPyUa5JZqluxy0Kk0cNQjFVhPcMumqQjMfN1EH+H7reskN459CE/HZr9T54KX3SMfrq5L2XivfsWlO4lMisZuITVLeikTK1jPD2dMG5gUxXWZXJYZPQsYWSEoltt6KtIwHTvT/WQcjSbpdTfluXibKq2DA40O/LCxCSUiJhnrzI9HsCsvPtiJHP56P39YGLmS20J2oKVKCwbgelyM09JB1EZySYmli6JW3WzoAs+i9u7cAyr5LYiV12LidFuHuIaGz12/v2YJ4dV5+oyjEgHv5qorSYmjH5kT4FfPQmnkYtxD9ZeYwoDnVONx8S5pmwyyQ1Skqk7YaIe0vC7L8wHEI/z3HR+cNNqDxKQo2pTAkIa+uoyfjiNe35es3+BXxFv7b7Tu6gYYLEHq2FYDDXOJiUpaAhk8QKEnwy5rW+CM7elGrbYxHP+tErUbHcuRSqHCvvgr9nTM+fMBGPoHzU7zzltAahqOCc6PQBIHDc3RrfPTOaIxW6F9zgWOQ1Ab5eI7TJWzqIiMYL5eqrunUruQBuWSLfA23uUs691dN/YZyediRzAS6Kirz9vPbHBRK972GxdKGhcwwsSywC5sShkrKdEkNrlEspLD85Afq5zeHpCh+hnhFDo3icw1MgFwBPPxFHsydIVFRPhDgEc0d7tlpS/kz7MUpR9cq7A289l1LRqeITnfHPXoqZ/FGMBS/yihni127nMV4IaWtN5jy6A3r1rccZR+Pg6RnWwhsTgyUiHMHav9TmeVyBKwNfCznw3mhrKvIzvND2rI7Oy0xViWSAhPYN8El5sjcpzkPoWJbCsGPmHE+ii2PxHHGqlvvWkIfTFECphntTBP9668PZFIihx3mRAsiORqrBLgdiQjP3TM4phbkjmTCx4Tle4C+KlhwUxJEmB6ojMdYuiLfUls/HnaFXDvW7QsJ4ApWWW5ohJGQfeg4f2/eYYnfIKVkSvt9HTtCoXXkZhXW56kVygDU9CCLhGtYbzs0eGhqz3hMlQwGRRlaOfveh5G+ypWEW4yfZH/rpzVq1CqvxihFe3GpzLbOEaEB8de8IbcGKT4zCkIvKbZsWfDpSHDPp38NM0b+4OfzlM3bIwTKVO/mnmso246ALl3EVA6Y2f6dsC1JxoQRqobM/adQIYNRKOlbRlD80gqNLTF/qPrsvbI5PjzSYgW224lRP4mnRJO9Z7MwQ6GUNbCjVb2VoTB6udE1hapxSJ8qIu407SDtIxN48/82OAZ1mqEjn78kzz6Tn0rxXIVJs2cUW7XKIcO7QIpRNsEvlHcoXK803+NFtZ979Me40rw5TYT4v9gn9DQC1jWXlVaYDkFnWhcdAWsI3JwFMr++Km/OEDFKjA7FftJdRl5nCusIu+teLJCb+CaGbpI/LH1AO3olxtO17BqYMrt8PYBK+1pzUK+kfRBzIbD+tPRgG9NSB2yjJZLNWSP7dUgSk5RVmUbJD0sobVX4w/rGwNKfIBSh2Y3kvEypEMb3EAC59sHl7VaoR0xO5c5t3Fe+T0vUj6T9ubnxC5bMeSE5DyQXwugVqQsLaUZtBbgpveRgHp6NCfhUivtvzjo007wERqxpBVGXTEjcZPFCVW0ig6jeqyMaCWSKY7dT71mczall5EVaFS8DTmKuLL+6SKE8FIi8chA9zcK9ihHrCxm9BNFb8/HgndHA0oflBCP2cU3d286Doz/m0PEKtlZz7o3hlf0waSErjrOjh5qXdVc0kFrUrsnl2Pg6dS1aPjAPHdsrrxIef6vw5gbZc6UXNrU4TTUbwMSnB4QT5fnCDGXMSuZvRMZmO69IH4cErGsM9KRb+/GmxBvUmbe0oc75ZZuOoGpqoVBRlS4G5LGN6VdeTlyZ50o1ka1h/wYFS3PcZ4ANHhQ3bx9t7fPqEZmmkXB1gEs3PC6soBqh9+NKtbB5lKnK7TX4qkczmAEWLBiL1PB0tTN9qrNAn6ouaw3zxw6pZXLWpJ+PRpkc6VwakJzUlI6jT8fs3gZpiB1M3LZanyiOM75v00CkUHfk/jKzVfpvO9dhhL9g4l5PSzdXnYSO5pF6wBpSmtH+ugRvVLyik++NZgvUGwhYUuT8O7eRxaGEcARF+EyUjV3IsE1PQ+gZGGtzSMVxr0rlP3n8UkckCZj4t/F2bS5wLYjlhmaSykYKLXvrf7dz8tZaX7gtKAS16uCM1gX1cRB2WOs0Qfwj7kIfp8/vX3x804RE6lIyD3bpYZYSP6pLJvH9iGxsHoYVIPgBJU8k6BzNiOKppZhB+bSAv+Aoy76RnQ8npUHiNmzi80ki6nhkKgJwm6QFS8ow09ZL5+oOCNBvWwfITFgAU5DKC3+fpc9bFWjaGEODBbnNLhkHod887nI+nR2iD/lPKFTlJg12aBdAWKWUHnbAPe5VNmRhfh7HFgNeuAKDHJAeEFe5tfmPEQBsD4POh1Zpui2mmSCvqSpppEArDxcUNbLPF7SYRlIHXYI1AEqfYbgV1Ijo6IMoWXRxTsUXTSQukW9XGuq1w0xiFgHfLiLDR5kStYaNnHjspdsukC1KCBm2rjqdanD68snvsvIA61QbdhtswHnQbE0bUVj7FCAJ57JMrd87KbipnnDuSmSaSPj1FYOSolyLXxUDMGeiiHsTXK2XMjW7TlLKWER5fihnqEmzPnV9gieFnOOgVTaj0iAfFraPEx8GsBBfrYEb6ZoVHNVn7k3ZuKMjUV8XRCqfhpGlyrYJFrC+aFM2McdM8ZvdPe8e8WtCQE3Xn4sWbb2DxfCuWHEoJzwjPi5HbrVAT0XDFHZOR48VzNjW//mUeCm/KmIg20llspdNvxjYGAM=", + "page_age": "2 days ago", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + }, + { + "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOuHuK1JGxrX6+zclxoMWAc4pbopaBlNbkj0IjB0E0n5ijsZ/3q99pWnuVcXKV9fzZq5UrzvBnHK6qieyO1DYSEB+SttetNZP8ggYRcq0x6/Drd31D+BONbZ0MTmUMdFN8LEMNvJAoDDNEjqe5JSWdx9JyDb28InYj3ze7FhY7mmpTF3FFv6e5W4wSH2qlOxQd6f0DJsvsihlGaibpIeFIqAM3DWbmjOYPoD2HlDVVv3C8laNdbChUEwvvt90ISKxjFdPQIOn8pkxfEdzCE4Z1JdCkSXhslwIxd1x/FOpRhC3f7Bj8eILUzppnC0qgMo14VTgmGDZGJcgYLGZuWyk7gWdf4VMitLNyUyygV45eHMtlGcMBupT/HKXpKahj5YMXwpCGM1Ov4+eNaU0yfPzgl2cJDsqcQTq0KblfIPBuU4TSpnJ4mwChYVoHOwmeX1+DmqL1iS22Agp3Ycnqw0Qsms2b/95/RqS9x1V1kE3H9iu4VYsozEfI6nWS5qW4HStPNWKUb7r52gM7ZhD77cwpM/h3ol6459Y6ajjfZVTdrsbBqv6xvukERKjJTuB+jRwsqDTu8+tqJx3OajIW/V7dUFwXseOMlNG5YM7f3xBWRa9Dx0LaUete5c0blGGRqtIwfVX0czGFp12GK2Qned7M4nsEEzvHGr7LVXoc6sGr5hnjaYQ0o5O8hPn2hMhsWHeODTl5KiSt4dDlmGTjH5t3tIcNUqojzWuRRHGVjINTuhiEAsbD1yLfEUNIm+5G0oPu/luGa88knS+lWUO1ZZZIA5bzzlFkhmx7BRttYmsBDL/bjM8DGpa+poXY6i3eizRHMrQYgwCgBiE+uuBkzxJUJU6qg2p2fRw5fY/9ynhBXIvjIwxVSSo9w//S4NjDJeQZPuwwLXZh9BSgj8O+OhuPUYJf6n+yrjQlvvNE12VIqrBauJrMPmFisc8H4nHgYVJRpFyoaulLVIv33tj8ERzumigS+PiRBdJT8kPle0fPLkgpNZJ3BmBVzrHn1576U7yim43WkJ/oNDTqJSUxKEY7l403W+HL27Jr1Y6tt8jYk2m3+bDwpJnR7Hc6qvv3Z1fqzq+675Eb6I/h+bQp+mpDjBoxVYrgafiTA5bFeNNuQfloc2Hrz6QDdqbtXZ25eUrdGMg5bMClpYtaUM/83zJo+L4iTYCUcczzWy+ziTPNxd+DCqKjvEQuGmBk4SH3Wu7zLXsMMiW4cSAwMi4KyHgsiqV7gwZv6H22qFJ1TBUCC/kwFYo4sYHxkDLeamTzm1Vi1vwpiUG52ery8AcqQ3i/JwdwaJr3RTb0xwR/3jrBvaXzIrEzD+O/LBAZ30t4+3VVF9Dq/7CwLU5/10wZzZa1L+9xT3nXq5ce/mP0Ww/usXYVsPu88PGq5ho1cOc+Wp668RWz6mj9svhErboauWSCofNkdvHvD6fC50DbCRk+r1I7KCXaJPuoJg3hRuA+t7Vb1M913uIPVZx1rDKSFt8sDPTsz1rokpaD2r1Xpe23B8gWfFR4QnZ1shTKuzlBQon6kBWMqNozFLXLP5Zhmo6hts6ouN6HEE5qllKuUIFHg/S8xOgsGXvptXmimFXDcUz3T3oXRQ0Gn+OZN1xmx8LImeMn8fIRVCB4V0R8py8bFWyeFRqaQP3nWwnDdMXWcPn4bxOhWndjFXyi1wMMUog2DRtbFGepNGhcLQcnk9pZjns18FLZg95jnGCtBnjoylMRS2bGX10x8D+WTYFvjy2uSTeUob4JH/kaVw45UiJU9pa0hr3t6yyb9CHc+xFqzEuxA17+kPrw8b0rM+1Ou8dkNlPxMxGhJGyF7V3g4Vta1WB9cyE3Xl2oFPQMD/MfzG3ibsiCo8cSHT4QKzUocBQwQCfJmYf9fs55aX4Fo9kh7G8Ih8brK36PWFCet5sPCOKfC95DniP7PbUf51DsAWeebuWtWnrOOMtMEkwigol2WsstaDVi6WGgcB2xr7S5iAjKZRF9tG42VtIWMLFAWLn5DhJiLrpQA2KjGIvQUORROy9KVuz3HcRkVS9vf54H3Yo7EuePgQe91QkMvT2pi6Wq2Y2yWieBxYuUR/kuLujnwNJoZbTQMbWtkJ1wBW/sj7tMdS837mT0iTUedFkBQQgDk1umz80f8fP/JvkscQhRNmzXHbC0EsmAPxbBlBkA6LyeGKO4+T6qO+efwaDuZHlbo/+Ad4PTB7FgLqSNvMpPFtSiyK9i/vdrERNUuYmzL3FFvs3Yhm7g+Qtg57FrBvyEiDq/BWoYB0csm+KVvJzaQGmYt4EkDGfGzS7UHy9TnraykNS1wFJkMjlutZry7TIBTLqfqOBFGfQzzi6Nyg0skh6X2Bbom5tKQ5y5sqrBPMcZIJaabG951LRR60NYoVfQQhDR3pjmIupkEA2yC7PUIxjB0EAeRbnw5AxLx6HRhzNWL1j7SRY6N6LIC0Q/BdFb71mJrxo5C62xkNBbnxZ3jr9rrZLZy4L4sjvAB1Oa37oNzWl+JqoDR//jz8g7gfDwn5rqN9StlrTbYg2el/jG0a1N2wnX1zQPYQBHuc6SwRfF/Nis84T8grSKhXdkBD5Y3bueIySjw9BukUqxxUHbxsvuBTpg2Dg/Y7ECN2b+M7H9R2q0FlxLvR04AdsDJg9nZVdTDs5EFwnUBWclPLNiKtT1j93R81Msg6OBgsZiIRZtzPHOKq5YzUoYNbfhdiGzEfOZgRMj1mQd5YF7j6ps5B/XDqnSr7LRHCHZKwMmnoH94SIhfhztbsACzayiJsxNlkzxMM96c3w2gQmFi4kKJG5gmU4G1CmDA1vXcgrf9XYuhdBg66FkMYH1atthKUMz0w0+XwX18D8RnNPxa1JUtQ0DdZ5p0V9AtT2YI9RLGtwz6vw33RyQA6c6f5nwnm/f5qPCdLFLOexXTMcX9kXG7Dc6rE/RfBz4lZU70OsH2tHU9ERAt4SqDzNorDwtTikH6RcIVVO8vW9MGhuCMQRSID8lH+iZQBOFBgO3YUo5TzSl227Ua/pqewxY60+oq4g/8AOEYxqG3tXq61bCkWDgnMKR1EDco9+W56rir9RSnNHLjNF/2lOLWyY3zD7kwg+8lqA673oPiZiec6566aubqWJjSBm7GUJXlGQfqthsJpcHL2DRIsk21fvkmIjw+qqKr5XQVaPdJpD/TA0Iw4EyoUVER7Cnxz0uHYZi/h2rilARzx/eqOa94VUqnPMDqtGiE8HqWHtsoNjDfXA08SlKu8z5m0/FcNnYmWDrv5IPJxbKaketumP2ZW09N2EAUrsPyYJZpKCniG7wSqQmYLGcnP1JrOnMS9eh6XSN6AGu9WQzbg2RZU269W/hQjt+j9dIhN9RLvPK87JonKTgVPNUdCH+BYvZuJjPHH8AGZVQZuIlMIfTcKtW+eaGHa6+bQO0PRV62J4L8INcYryf9ivVi+L5HiLzrMpv67ZsCJxslgJpIlVJU1dJKhZoCW+kbOEd4KMu1Ucbbg+RvKLfRi43Dj+c2Kxf2P89mx+yukx8zIVNYUjwi17khBS6V3RX9oIyDxI/23s2SKomout5rFP5Jlt271FKHbs4E/W79zpFy3WIl+ydwKyvzaPXZePeH08ps9eWamiP0OppWSQIZXRtMOFklFc3WwBSWr0EYNbP1zOKonnHYMTWHPjXglBQHiqNJVO3UlCT7Ozz84sJWWwHTS3iOZwlaxdekwcFR4TKQcROuqtA53Mxy4LKhPFVgRvFcZpw2QcIDYeXWx1xl7ewJQ1DgzQDDKr/JwSs/tCvZmq5VKCAOeGXUuuRwa9i1l9bXAmTvBQZ0Fbz0DmY+s5BelpRoNM3SE7zxH6He3VX6Wk6rIOmUifupipjLnRpKQ3S1cQO3/npzxed9SjNONdc9MdzusqZ/7fLHBITAXDWRkfFFqDSSQnsf+61qNkyNDJY6yRMDeQQk6hPSuWv2AUZ9STGS7t9UlyqJbrnr+B6RMq1fFQ7g5F1mCaBaBWvUmxe0Ktou+G4LfmZRCoJm3ZFPj357S44XI+RBwOUHRxaBXSLSOVFkyHAhA2732W+GcrCMpgzNwjMVlXjN3ZegxO+oDUnrbKf1jc4+MAdYi2j4fPOsm0B+CvAAXJ2taqBX0JWkvC95BHrlz0AskMiR4mILPSCPTee+ibaEagDg7TUfgcioEGq6b8WFJRHgiHha2i8WOwNQNGI8RjzQK8BvEi8yL4qAfNihWLYlLVlc4W4tlUe7bXALICr9RCKhRPzgJWGGcitHPohkamEPI+Y9vlg9PJX/fQfDYZLqqQ309aljdb6xbo7kkjPwcDGeqbbuhRqMDO3M85mPQBQZehnHq+EJjDjABxyysuRKcBJsAiVIFYNJRMXxQmXvbGYyZSSd5WT2FYDsp25UkDeiyZK+nCR2FcqycAvsPp74eGi1oybgvNKZqXqR9+hu2wlZtv1HG67Z+eIAPhY5dfplQKaR07IS5ZQFsTUs0UpSrIZO5Ju4cbaG/3cpBZ+6wvnA9SC912vtAnz08618cSrCVdUb6IgvnXEf8itRtPWASqIQH95+B9hvDZLV6xI4T/7ZPHaDV1PMjHzVIs0UlvvTurZlwpcOJ/e+t3zUZEyr/y6bIU8m4MNx6Mmq5xSNQ+ZoGTi/hVTMWHfCvuJg1xwfxUsYdzhtH2oAXvXmAiGAfNLm61lvuSH8WXI26yyIiEEs62gPWkPcjKcTVKzvJfX7tg6fp2QDE6O9LJyzsLXC/RZDTapZYsTJovzZpJ5xltsphp5VfhGfDNOU0jr62ML0Y84A63ShNdj63uEFq4nqA+SQaI16qOZZzKEq4bpz3XkA2lZOz1xmmdH0u9eLwiCEGda3Uzk/yQsAmt4N1daR5ruc5R2YZ/+EoZ1RZhecv4Gl0IUL4yFhgpAl9tK+DOAsyKaRhu8geVHwKqw/8VQ/I3LlPLl+/JythWekewobOl7QHrDUXxVQlud8dCXi1HxbdSlVCj8Z2Q+2sxZ2jV5XVbkJwYo0cR6P8j5jBu65nh91CTgmm88fZsZXrYYro1oBJftUPJl1nBAgxXcp5WPsHmyuNWsBmhzuqEma8Kil6Kws07QTdWnw1SeWrj1f9dyfOa8VqaBljq4gYLXRnfnH67nmweZdwWDT8pHb5IAfhVUcbjze9nl4LQxcjehHH7rhvbcNc4X9vz2TeuaIcXoXO5F2IdkI62tI//DHM1ZZH2F1riU3Uz6GcTqA/giLQPukW4po/lBG+WerKJBsIyWyqknTvz8RaZ8Z+gLp9eKlv+UFgQpCDkbv+SUHcUG2MfwqjmgbH8pzpY9jxj+pQCvEheR6Ib/5powRQg9BRM6sGofOLKzVu33DkeRgD", + "page_age": null, + "title": "Release Notes | SAP Cloud SDK for AI", + "type": "web_search_result", + "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + }, + { + "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAT+8zYgTM1JcJxJVhoMjw13uJThlc/AJrmKIjCyvetCNVP6qITUAN1WofuzWM4tdx5qWDkbqoyBhsOSuGj7k31lRepGz5O06t7L6KAqyhQ07e0yFeHDevO4JdKa9OfIcn0IvrM8c0dWooXhW2EN7mALE0Cs+t4jOhElXWRNJyKX/0OhZZcgusHOJhkCwR2bHnxwDLkVrWL514ZddnWpNu3bwA9pTakCqSfgWVVYzjJXNL6unIo7eeupGF8kD92e4Bv+HGB7TUVolPARWwrG0oG8FBhSP6Mm0INFlkPT6O3YE94W8ro2KsJlg39zxvXJq1kBCr8OBfvyz7Ck/+BRD/4oGeiFOUmSnmoq2tC2YDOflihYCCmfIh17uZgVeB09DPOxSK0N2UuAnGEs9VYmYpcszz4Uibj/7ibg2ejJNZR5Do7JBrA8OE7I49wwEDOXef5AY59GFDTG64eQ4UW3M3d24TBMFQu4mXYELDTRtkyO7WKiZ1eoWBs4whsouDBQu9YvxnxPsTajhR/S7NqhLmfpMWlIxmhoboEKty6AksPmAnFPuxsbXx3NuOpe4VGJzp0Xuaapt1HyWJjxGaBRH+VfrFldEbO2seQ32N/Q5FBPl3nfULF6/4EKqzVzyEu7GytzF/Q/FpU6E7AQxYh4G4DPbx9oUSJQ+khPFpttaROl0+0wkq5hrlEBHkAxr4qxH5ibUsvVjITHLPcIPyypGy+L6/NpS2w6jQon1ceG/s4/TeQHrR78ZvLcew3GFiNQjW8FnPzBeAPoGyY/ucrUSbvkcf8PNh8o52Z4KYudtFeva6UaH9GJeWAvXJz3vbcYrmcYts1HpQq6FbwoW2T10ie6ZphTCEa2VP8iTlYAYE+WRcEwFPPJDzoEwXlS3lbFicqxszHwRe7T5ltsYhZOLUYK3Kq6tmgM4kZ/CUN+qDJyHPTbkSUjZ45czNoZfZa5vwuMpKwCqvitwRwVknKDnlqteqlle2Fw/w1EeDWrcuYHgPvXeADQzyFAnJt0PVzWisvuwON7F2o3DqTyVTA+AZBm3TrWYXoOOZS7nzDpWU4hAwpOdwI8BSVY9QJmQVDosJ6Snuw7e+8hbKIR4VrUl8N/3SE45zl1l+TJykBOb5fFE8MocuGjanSPydfO9pE3gZKmpYR4GFgIyImCjM8KrennrqYVESUE1r5/FeDYUV/kq7FGtLNsmo0Sadu9Z9ERTuKY7hxrTc1QSbOBhW1IKs9LfqbfI804xVI5/wKJuikWwORm/Yfc6ptK9CJh2kDon4J3bfcPerfemmD3ZvIhbYnzsWu+qUdqGbH2CDVZkK38qNWqME3QOz94R2tAx+PVcD1FWtNstreGc6vvYKLHeeJ9YGJhFuSp0lik+ELdwI3g8k5PEJsmzL6ncwaAMTDaGcg7eG5VCayMO7bcaiAwpL9ayXsbzxN6aanmTk/P1Jyz8TMfXEjh/rswYlYs+gC5NRfxGrdOPD7Vrg3BiGHPT1BmvJUQIRoT5fM39RIWut6yHSi2jm86wPf/PPXIBPij4FrRrJHu/MXu2w2w2doH/ch70VyUa2JucrC1WcqZIL9sojEEdVmHrqRq8RqGXF/74BNvSNTdl/3BKyMqe54xRLLNMDAOm9KzrsB0P6FziCAaxc/TrMBy2lZxcfSMUcRioyqDDhU69doEBaLzINUulbbk7e74x1Q49pnnNs9pW1SGVifvh29wZmaC6+zRoKqCIjYjyTWAPxxBhaYubXTsyq5ykMKCNNAVQIbkGjvV/69isx7hasOq3HdDPg4avyWfllZiCtq6SQcFnQSQnFbZ+nppejOEfW+3N7dnsk+yRIkYChVEJuZo9dha0dL0lmSVWoghzFOGMyIWO+jnFl3P6OTRXImxha3xTLbE9y+EK6fMAtuzV5gBPdO58L2W0JNf0lOJpxpveyxC+KA7ic1qYjOQQJWXa78qNdYVQevQLqXBC5hDk6+Za50zXA2y9YYSUNZaDCiTR1E2yk7TYziXh9IIHwUHdcTxdc4uXeYMmNAmT9yh07FXbPvuA6xU0GtzMlONgjSK6JMfcwZhFdnZto507WsFcQ2Ff0qS+iwNbeIM0YNDifiFUHxK1+pOauz9+iWawWIISOal/jLSMht13r58trpTTDY00RqEMBJG7w0vLZNQuwc0qehzeQMpwaGZIAezpPflFGHIxNiORpSvE+pBQXjdU8ILXjas3cFTDClcFJGxTrpXS7JNnbsVVZElZ/nvVrzJ9D3i/IuWul2I0qKY2Rp7jdYZNrpES2RnIVKAwUwmCWuPafiW396GJqfrUTI0SQgrevI9jPH8LGYjiaCi4BmPrXK+YkbUbCWoyHcQdhtK9yHOcpmye7aaeRzznAcSghap/77Vr+nNEFbSgG5Nzad4nfHXtp+lPWs7gBi9zWSRdMyz6QSC5zbcwliUALVNxi4MUu0xdwsK6LCgvHn4VPd7CTZ1z8JlvOJ9UO8vgikqN02RZve7DTU97VnBBr8JWNaWSAGKV7aNp4nGSVT/YaWv4IJi3jGW1HE43M9+3CGkxvAoqmeBuL6wdTSAma42auo4FteSRgBgQhP5nhAamObhZJpwwzzxTVlSeohVyLDUXAYu1fqGznfvOvHXqwYHlpCVVKuPsAI0RQX0e18anYyUQ8NIezpvm6InOZzVfbBoTlFx/LNqJKxyQbLwi94PBB9Xw6ilkdCFG4SVrNsxpPao1XsE3vSW0fq9YLp6TN/klvLmMgyfZXeYhxGuMujQJ4WeRhBfw27JEDVcJSe99ifCu+QY3cw0rN9vrmkuWi2LvqLb5D7PFSsqXFaKqvJtuANKdeP27iQaxU3EepRy7K+5QyM/AN0oSivKW2WNxq94mh7zC4OlFZPOeblV1SLeae2Vjs4Mp4Xka2TYhJuKEHzBiOUaMrn5tMc1oikhBLOXgPzN3H41eL428k5JuFTvrPLOq0aaW6LrpYiUxQSjMyAR4LsSteT0A+WpOzjK2ihG0x/cWMjnpPB8ccdoykR3Ko47zZTtE+t9uL514vRsDjmkrSQm0dst2sB2dhAppoN7SX3Rz3SW1IaV9xugtXjtUW4b3onbGRwuN6hsVa/QxGfT38bhqbRhlR1UWk6xvnH8jJY/OVFTsPAjpdGDIsL/7d7mz0uPvl8xQXTicYPdaSjBiVCbvVwtVUKjRRJrjUGtSIK1s4Z8pVRA4Q7g0ASizLlAhK646prOPQZSwIhLTmPsufQy36SAp1js1xVHw8SE8jEo3Or8p2wbCv2XurHuN+k102z/ZMkSfBUacvA1YuEoyrpmmKgYvoAHorkFXeySZF+ezBCPCVC4FcJdBiAe+zEt5mWGvV46kKNMa6AeuRhqxTPXzUp9pZT/RQ0c9FE6PTHaPWfz9NJaN/GYToEBslHu1Os0A8Ta/G2QdzGzkY2bZ7qZ40mUwkys2U2HANPto4oAEybh2Xke1RKvAJOFvtZ14w9+VWvk5NX/9uvx9ppaOSpwpijCs8j3ONhDxYAoi/JcaXWwebDP0FAWHnBRcCXNNQcdV/VbUx+FDKJ+iAjvxrDG1uftVkWM0b9+4sDRThetEZwc+fU4Qsu6bsNIFz8l6vr891caMzlSBsZ972OwBxfruv0YAw==", + "page_age": "1 month ago", + "title": "The next evolution of the Agents SDK | OpenAI", + "type": "web_search_result", + "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + }, + { + "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDFfN+Gdu0tEt3bG04hoMJrX0vSvbN7vJPrwuIjAgnlcb/vmr78XYUzBoLVYPe76T0mzspJkHE0lu3KegnPXNBAsqhEa7sLoJVDL3aI4q6x30FBB7NfnSu5x/6I03cvAsqIovNQXY71xbXJrf3Kyh0TAaIazN51YlNCNBYPBlCpoLnYjRPhq9S7ooT9sHbBoXWiYItBRX/0I53s+9tEDiDyqUhB8UYdf/NqCLJFI1aMapL17UP+p3QkCU+fRLtELQFhudeabA7MyFWs/FWImBqe7r/Az9dNtqXvjFU7k+aBwkD4E67k7M5BP1+AvyiymWvEhU8VdawKHxg8HC8/i6t9NbXM9lo56WjWr8ZGcfhbQt5xeZFdT6aaY9xSWJACnF+JOXRXwxmPRsF/jougyx/EpVX4CM7+gLiNOLYe8/WFK9TIgG621PU7SOM7hhLk5JSuVdRgWvxq/S3BFwmSC7xzWpFyszPlKQjhWXcgxQ6Mu4N/9JubWWAG23nDZvCsWjxUU1S1HLSFGxkUsvDMcNkRK36m7trEiRtjFqtoaAyV+rT/ihtwB9kXiCKQza74JppLoO6Qzu5XE/aIcjeX7dCbPPBCLjfwkRdo9x5M8gn7DeGXtAS3+BIkFFRnB+E1fFcTMqRnIkxjD27UiMopupxN3xUTLYVrYLtnbJ2MGW8XpCnSR/GXjuUlooUZmw16EarVUNn5Tv9AanCgURICiD2qviCrVAmsYxTLieA4CzsrQ/hxYrlf7bVJhS9YJCuqOpoLWsmhLQE5LKc2iq5RsqryUJYOSOVGHnCQSDAi4zpq6E0PD/or6wvlZe2l8yP9x3r7u2aUUOVdveNY8yzA5aZ/xZf+w5FitrITRt1qMFSF8d+IKi8EG1w3Ez41oolOj66ggKtmOY8s0JhjCQJsTyje+dlylJAlGZSpn/KVtyBZNiCIwyOwz6j+KuUS1sAU6dIwRmgpOntPoytgwLzoJcCM7f0USL5HMGPOQfrkH93qjs8OhDu1eN5kszSfHAgKto3nz17kSZRh/P4jEvcVnbA1OuiihjHmm+OeaaVEWFRb/3VNiDHDSaoct7MTt8JjFzA20nj6d5oUFBjlXXyejqBSgXE5gKtWYefl+uGMDUEXGA4VFOW+DxR8E7iqgqlOFmw0t2t7qCpidTmdR7i4dOPV3SOUZ16CzALW2+CoxA72cFFqpSNNPm3r0Vmk0GvBlEDN09KjR8t/0O3of8fi1949CBcKcs14uTdaZ3oiIhZKFMD+MeIPLO8r4ta5zsaCPpOKUYGzFLCXJ0N4BaG1X9tzxF+vTxDwqCWpsVwSuHXy8u4XsgyDGIx2+J9KiDvgPcaI6Ss3I+FstXWhPNaJlOFA9XItk2r1ZbdUFSAVP1cbastnjg3QVnsPQeudmIeMe5PNdeGQPSnMHCeK+UQuTeZZKB2zKivcS/LueiTw2hCyZBnN7PqgeMxpuv+Fd02M5iSdYTTAQgeI/jQ1wSVfvNTxpD/c9xu7TmIjnlXQl7Alv4C3QJW6R/2ZMjwMY8XCRBDfaujGBeaNiE8ddskUK/uxV+bJtTRSHquoJ/W/oN3orbxBSHQ7hdTwYpfhRG1zzQ/HyswgFsqd/Z6zGE221qfYOnzsJoAzhxYYKIxWY5PrGRWQcNEd8PxGdz5dqluXt0dKgPW8urRI5MbftA1Erk1ZduLn/wRub8P3L5Xhyisgzi+tqSgkHeWfLCEleP3gcF+ugbpBnzSUjc8CWWh0YrPv2d6dtfvOhJWcgLGDhPbgVRB7LqaiFaa9k8C/tFsGD9hf/ni0D9b7ddc+iWav5Z7jRa6i3IbEKmE7Hg1ULfT+gkC+U/xKmKY32zrxH1wysPK5/mbcPVyli7vFi601fMnN49J9Ynb8xPFbcNZdrR6q7QZMcGkiFajDSuNha/QxAZLdXrrOOagvw/MwgcijVSQ3APD20BYko8PFdUP1HfErmwdB9+SP2XiwzrB3J/PMF3BLYSElLVY4Z91HAt8Z6KGS+yqgXyQ2bIfjLuaRek/FpX+ffj0vV6lR98cJvIE44cQNbixITrqVFQkuh3zGc5bWNMTnva/i02JEaoS2l7zfGVi32Xpoc8d61a/Iyww6sZHUl/BTv1Cj/1/h0aBBjEt2F/eE9+7bPme/6FYTomEStuYqz90IEnLTdGJq8hbZQmviGYGdSruHKcTCs6pfVMBaEvY0JrHwhNLtz6a4ZaamR8Lf5CxTT/T0AoTtZG5iBs43F93kDgyw5l1d2Key0tFffQPNwnxakJyPWKzdUWiSUQC7hpO0xB22n72ktpSgUdJ1MFx9O/muLPwBigUCBI/QaGyvreh67mjj+9ZKhbGltaN7gX/IEdHgyuDRI+XkYoViENJsT8iUVIzSy4KRWdbWED9Bk5WsKwlWt6/WvkbtdQyuNLXqsnlTIuMYNNQj3UYUiwqcNmisaG4TWJ8zziLZi0hy7jHHl3J220A6j7PZ8Wu6Xffb5Ii8m86g4oXaEkk+E01T7CJWY3avWHOEJzZ13W+HzVEXDz0w6exueG6XqYe1rdnd/TxMXZXjNV6nfo/dKQq58T/LdLuqFXOGvMFkwGzxZoo8wsddCy+hL7ZyINnVWpFlI2/M+yyKSbBfLVfVwuA7t2wdUcBV22MQMAQPOGE3iQXm43tCXKIocyxOWwGkzzMYGNojwwfJSg5GibnDiEWZEix9zBiP1P79WkIsOvqdCe4RHy9mOWau7SYz1POEjU7ndP70rejh5lp8vpYMdE971eoiub4G3Q3O6L+WiaOmOQQQzO59GeDHOF/AUnp72zzR4O/6pNp4nAcMPkyazFucJ8kW43n9zUlxob8HPrztjFok92uBKDvb31ZPw6ifZd3KxCXMv7fZCFKG+K5/+jCNYWx+atPr1s7+6A5ZatnwyiN53ZYYIHytglCzLzMJGcej8h71LY81YJBsBUidM612K/OBU9jYZfYt8X/pxzB/U9EJbHeCZ4tqsVlVmKvaXPhxM02IKuE0wkj/h3Xrvavy2g66XqU+HGG1GxNGLGFnI4MeNQORJ23ASamA+r3gqiF3u6/HYkt+rpcMCZtWUZ/IgN3GUEUU6NVIO7qPQcypYc7uURWWtesUzKWfyV7rUdDth/sD+EmIio0KB7VRmDWuMT4zIRNx6zCEpQzZ5KbuKIm4HD+ACI6BYjSkdVPXdNbOTk//xZ8Kg80CCF+7e2jBIyx6KpLTsCmDmH6nQ2+mIp8IS91d3/vrSHyzzK1Hm282NXahSWZWEm1aCkD8H4DBWhKeBUze15o9j0DWEFCPvI2hX0lSYcOjmp0azGa2sIiTw6bPmbKgDTvZqH1kHqTfDQRPGIPW2hV6sItomYECFcpKVaRDZ5e2M9N/FU/cojrFymLJLGbnkXK80Zxi6VqHbx+lNVChT8pr1E8xINldjOiPvOb6OmipEnMdSRv9UlryGY1WvyI4olfm2QaVLSooo5+FTpzNijqOwF/PWoSjF/IHtjmM+SrtrTdi7CgrLG1RcQnwIIMH7t7WgjtiEVjVOpFN/S8Gk+Rcc9RykkFzas3LyVc99H9LsgPlg+eGa36MfTrISJcFjxcV+QV6/+CGGJHePpX2wDSdwVn28ajuhAVDT/WGiTIZpBYHUdeIcku73KWSf52VxqTzG7FbTlg4Kow16qwAvprNIIxGtWSaM1Lnif2yDpXJXYB2SwBUFNDMhbhPJk/aGoPgOdmtxBb6Qd9YkwgeK3I8QLfS+yngwJzFJEJSMTBD0HAsm8TSY37blTrzeJFeJ2ayRTka8PVvlWlqmX55ClgSQIyBpO2QSbiUJSE0SGh7j6Q2vpP86u5qYdSKp6evEhSdfJRbdin4eW+Bxh1uP09u+At80T/hlbrF88K64+46KZL+zZn39NsVNVL6T6TzOOPwiMDW4TAugax66wLdQcEw+bSXz7C4CFH24WGGbszFJEyHLGrkrMZrkuvppTUgkyoS6wuhkO16I2GC2a2iVLS1gCw6zFAZEKg0kEIUYaOtqmD1H1zhHZkDcvV4twKR9AQ15q2gfAkRiWeCzV6fVzyEutGWXt8Pgzq36gxJADlgGG9ePOyUci/V3jlXv1xJ/KeP0jWKmrp1sOgNl4aqHFtYqJK05ijcX3j6rSgMwop8KJ59RWPsiKChmgQKj3D+KNZFsyXTrYayj5TszQY5geo5KHjGOObWvWTwkw1uFPNKDnGb9k+NHiXp3/radd3objg/oSZ01ZnUWB3jer5dvBDEcl8buFqcZzElaeJ7dvA1QrCU4i8C1N9Px43BnX8P1bMKgVFwfXeeWq4ukciTFFakCnfJsQb640enUCHgCzjAx7cBBzdILzMiaeYDlEF9k8cX6SQCYiSPaftKbB9sZoxHFaka16t0gSMX0szSIbDUpS7F6S33cM1M5GIQi8sHNWptDlsYTmXIu5zqx0Wm8MlZq1EXncNakI5WGiy3HLpz2EwVUq/k5inQAJLYQEQVPOlioB0T0fNkqxj71sDgidaiFgE/SaRw7UmofhYPtJR7jG2yuGumoYUOJSk/99N3+CzqbqqfFHsI8E50XY6oz/hPqOYMD2mpYdkcNAJkufOJhLU7+1UHdPHaq1YAH53sgodpui2Tm3SRlqo5z3EoCjsBP+hGtcIN/sgcQ8kp2MKk2xk74JDN7qTxGImf6dAIyVsTUf9caNNk0ezdgOKb0U4YxmZy4QwTCqss27Drw6Y6sOlp2ZaOMyBKNv246917qvlI3t38FmGdqATnUhLnbP+ziP/To1kz8jaJo+9wIY1l0L4Iy2mBnUj8k51XqG+5ROad66S73nNaanKhpslB9MsOKSsYJYjlHpsKYKEKgLWd8qsmPD81jvW9wSV29uyy38I8a+xZOfEmEwM6itlykZOfo4xPuwjUs6WWIaJ+9FaULBoBlDMrDhK2JC+7bCsJTelWAN4Dc/6KVBF1IDzkHqYqRkWCOJiv3+bKlN70oTcE5Jg2trf4jIYStNzrl8xACvXqD2KCy1MtdxiBaH41maAIdtsfA0eMUF1wLCaTrAd2x4NQC/5a4WgXbZ8vODc5a7rwLfxYnP/SF3wlMNf7OEIt491cFQLQe5zALfUpEiMB2ELzIvn5fkNpNvspFJVu6f7omwIgnbmknFS8hmf9+P2rWVXXU2zKy44yrEpO4c1Q1hdXqp69TtYZ4Y69warYQlSzOW/MtnCt9WNGOuTj8YAw==", + "page_age": "April 8, 2026", + "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + "type": "web_search_result", + "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + }, + { + "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOBJUjRPHm8Bi6Qj5BoMMq4p91nQCL2rfO/LIjA6rWYsI/IwH8NSPfxEhRHF+I7KHbbCxRQldIEjo2QwdTn3+jgu6sefh6CaoeIB3/8qhxHUirJJNUcgoTvHuzO6MN6DzAFcUCUa/sHJWNXRkM4aPPMm4rIY2PMkKJz+e/FqjQUu/Z7VwNAmY3jK4GeoiPmxyMLoVmcX2R59BfSkePt/kIYiPUh1T/KLyBx4vRuzDjS0vfxnwOpkG1XEuW/duF1mY1aQiQuARNNoiM+bKVGa+Ukg8voZoenyWB0ZbfDnSDwU/B8jyvtM/74cW7NpjhEK/2RVx8xSV0KtRbPPpUXhuu5vAYLoCSZ5a92iM5LVg25cup2APVPpI1pQ4zN+hmlU5H5aIrl50qzYl/xoFhfLm0io+pAOb79fM3AfExfeOJU6kaZ2m97lmuWh2C+acYdz+4kUGxceNB5cUFS2NBXvxTusRUXCL1VP5laPOHyeO9uv9LE3QXZeoTGZFwlL1Bopr9cKHfhTSXOAj47CHegNMs+T2UYnauUbMM4c/8FS9tnM9/yRDtzZkgAVthMS64nzpekbAdbpFUBATrRombX6uSmTewSzsBgXU7sOPBJ/L7UHdHR0CemNJuowMWW3k2NqQAjf9FKzTtrnhUppsSwC0MRq8cpZLGBNOqmJ3xO4TYdt4nZN+Cpu5+OD3YyNSHZ/HQRs/sMZlX+Anh1cVBXv8fhVb4X1AcHfpn3AQuq636nSbRo12JseQKh/sjB0PQb4+f0tIz0ivC6yxayx3AfI3Co6Q66Jm9WXxmHNw2C0pvXX7jRCoJPjedv6INHfk+9/iNP97qhQ7EQh7rEKbtprpNVXN6liJW9oKafVik5+c2NbFIgDX8FDXIEgyUCrF4HOKqFELwRJ3+0TJsYJjy4HlyXku/cNwPBwqFG+dCMjUBt33m6Z3ZzZqMzz1cs4TwQQucm+eNaq/QyfZ6tRpuff4KVDIbrW5gTkGkKm9fVK/QvA8pbmyvZWQnvSZqmP6AgQy9My88aZBdrXrbETe4crvrOj0cVSrVbq8TBrIEgBGbIUzDEEthxWFAmKmare94Z/F035RUyx0WKOpET+B0cWzFKuf+HDFNvCYLtoVSUQarCM+oIC2fBURpURGnmo75PLqFC3liQtrS1NeBUo4sAgGt1BlqaHkbVJBchl2NIMgjB0ElZd0PiJRQD0s9Ga8RcJgF4Nytd7K7EWRnnSj1CYMJKEyFW6B/3AcjaCE0n12H6Nf30amZImSgQ1LeRDB/aiXCBDhwnuak0nhkrPeH6Zp1jqxaKQYfyXvnyXZ+5w3Jqx+/2+3jKmsWpuDrDZ5GiFbheP7cwxq9dcGjE/0QL9USJUCZPMV/H4tdWXqXsWof2pzqXhIUgh6BsDp3d2Ss6HbqYTddMjem2Yc43+qxjecFosq+5DB3Bqae6PzAVihu1WCcl/otfs9D9meIHpMbAS8D73ypaAi64tzeRg/busWl2tgtfAiroyEkScIbfmcvezaizDosXpIFnCquqHL+ZcP5CBvMe3/6viIzJhJ3nFB5VL+Tcm/JI2bA+SFJI4F6CeSP85xHuFcxrb0EoU172qO3IH2kUTXV8GlP08C+664AOOVDKOi91UqUwsEIYrOceNARLjYssUhPPWqx/7AOi9BrxfEfpVbyG5HPm/paeUK+2oCEoJHyGPCaF4V2EcZRlXghz35dOYrQ3SjaRPWBEzZCg1nbBTJpEgTaVJyjyu42LODcFh6pPqoI191KZiTyZFyuoOhf8DHA7P/mFy6YF/sPYsYu7JJn/e1mcxCwzqz4D10FHeqCvVAUGGyorAE5w1vamiOSrvne4Z4gWAtXcANTyNuKQABpv08kcPe3k9tXKK8N2pUiRvLJJeBUmCOrxBHfo6KpQG9MOT/q5bM7lqEEFNw+GrslkPufldA0CMccwxmxTC3WO+gcag0nbhwf3o4RwDLVLg4buEEcKtQd8p7zjOeuf0raLf/Tn2+QRnnGAzrqbYUboWJfM0AqjLRdG7wzc98QRK1nXuLLT9p0csPbd2k7IwIUh3Nn7QTKZURri3i1tlXduhtJ5Dl65Rk3XvOSIk9g89rnLswEXtpMHfuxFRtQWLO+tzyd8Lb85KDw+zelwZgCcVBmfCjFUOm+5Akd88Q3+0/F9pusmWGhQWZdLTt6D80pBwU2nsDxc8W55tyq58oda0+JkPb0CtJ79jC2o/DDx7UdI+o78s3Yj4dmMGFtpo2Qzb3LRSejUNlgrN+EO9MRGFzUcoCFM+mJ0h3gq/87eaJDtrp6xIhkILwwuxLwcqiZOmevCOL9eUTjX7s2srTLZZr1MVmprOisxUXT5xqfJSbfvjKBhpeTcULEqUe3IkadEgCiEuVVTZNjVqzbBn3E33bIC3Wl7uHQPB+ImVErxojW6XiI+gKeqqbeDrU+r2K+8IIBBq4aR6dWW/qALy14ERQY3BQw1EpwjdTe8blF2FZ8aXR6qI76zwzg34pn2szhJFTlP5kfBmP0nhkGVqCyLHnBpgwYH1q4UA8vKe/5TpJ4sD+DD578dTZz/M+dsHqPH+W/cAg8iqGxT+St3fXYR8ReMOFws1H6s0JrbUi2uL8U43oW8/GYjLfPieN1G0c/B1nzt1qAVVnrJUjgu8et3AjfjOvC3Z0RqJ+cB1WmJ5XJ6ZzZZ2IJaIkCbbbDMnSjVlgcoZzp07E519VTOYj3Aokd2+/9iOjllh/Zud8znp8sz5lJZskGCdwizRxAje/EGqnk7A9/91JzL382hKegG+yFyJfZJVFJf5JfMQlCW21IK2Gd09hyLsEnP9rqNZ/b60WR+ztfrum3mpHckXvzcJUzzGBgY1E5iPEyWiA2Y1sxp9dorHpmz2K/sO7EVSs1tfLlEHfokHhOc55Xu7N/o9U1CraQ8nd0O9vtGUuBFJDX/1PMwx61+KzaPT6T8zRRRFLrMgMyVFOe93tGQrNJZ1MD0AYl8YljNHpKj9A03D85sdU3G+mYo0YMq/KaYWBxgD", + "page_age": "4 days ago", + "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + "type": "web_search_result", + "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + }, + { + "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA3wUbunuwEMie71hRoMPfaxkZfHjC9ioMCZIjAleOM0fUbI8y59mErH7so1QdEUGUKXzR4MfG26gwYU2vFInP53877vR3TkVmVZMAMqjSDSqZNaCk3grt2x0NB4EMxyJ51MbOMHZPGacIrqqPm2sg5ki2noyUEIJlVwSDMF4tiQXFOxIr3EeuOe9h6NnIKDYTCInTmNRIAHryTxHq46cxy+Y6IuTZn94cD1OCoqyP7yTpzpKq2pCfSu82aBXq5QvukXaXz1AJDN8PEz6506Aw9x6S5DrZg4mfn3cUoDQv+IFzyBQ3IS6aiRr0LBmVli1hBhbgKbGh6FcAoBzHpupqBFi3CTqiUndMHJ4JuMXVzEVyE2VbKTBkiUqJ02q+Xf24YFndywCoP08yxLnLE8X6xs9Sq7NdZSkEBbSbP/5fBK0VY+30Txm2f3ppv2rKv8ROIxAAuuLg3tZ/m02EBz0WUt9qCGTOQza3sNoOe3tvyFOPyc6Q0YFIceurrotcbZHY16CgcmlfF73ItmymMUwhGtmJ+f5DShB6ETW/ukA7qbECOs2hWZWHfuBsUR4LJoNEvznNhihlLW2luLkIEpWwWX+eeNBeVFHj1W4ZyLDxfI6tvs88sVKynxP6qN2ToQDxvySN4Muvqso8ntDKfdOq/GYwhWaxFW/yg8tN0+JJSmGhrNx+40PfaM58EAYYU6rPgdi0ut3gwKdJziAlS86FOzFwIoyl2DehboRlDp7/HpuKKSqb/Kv6mMkwQBVG3CYysiLBAaR43tCaMYdZKW78PRS5Wc5KDkrPwUVbY+TCdw2urVw5WHCYk9owUb61IeFoFV4QAJttgW+mm/KcokHkI///P/pCji1jm9FPAvBPZRKWTK4Qp+dxedVRCH52LcXeHk+mYlOKEqXd3Ska1FfAXSj/7FUu05XJt2Qlk+q/xhZwmpKlJCHqR1EKoUoV+WRMg53CQs1k8Rr7RxxXdaEyds8gXDttROMbNnYHth6Ks4rB5akVHqVXHydePAvoicduEbPS+AO/EMfZpzqrjcxiC2Ac1vCpmA0eI4bFnlh+qJltus1aXRJ+tp4N6QvpY7vdxC3uOPZS5nsX/CzvRN/WyN4qX3JYZ2ImkDDvOF2SZldUesHh7i1+0v0XEhwY0U7F0WSBFXFfXakodfN25sZ4KZQqRJsnyFUp15TGyHpzh8KPyFSuALPbhHSPZvptlDYiJjDaJGoY8ZnAQaRA2peJi2lfZlMjysPtvXf1VyMk9E79pVSTJylnR9XUsugc6Ik9cabB72Pcx4wR/b6Vy2OjW2ZTCk7LhSdz2SFyvTl58ka38JME2iyTaP9b/tBUEFZ9lSAhpj0krlyqdt/QzFJqIQX7ajK6L8p6u5ucReKMVy/LY0l3jHFJwesO68CKz1/G/gs3Swxcpnz3tZsoPeBbuWiLj10X1f9kqb7WXrzbd1Zhp8ce+ffdqKcj1O/MUoywb/3lL5Lh4WxnbO2nun0k8n2AHaeaihUDq6jbk5YwysY58HbZevL3ePnXJqx5hj7EShEcGWUUIe3PJtPouvmzfNwzP3LgcAVpwYIMxNPNTxkTLx5CBcSsZWVE7lAOk9Adt23aRDBbinjsSoZhvgnJofRGLPSAgfOPYpmzg+JH8WRLsA84jXwD3K832NOyuJdrAg24G3WsuwMe/ePHTR0sV9hS8dpGF4SkMCwraw3FPc/pSCDYqefseuzfWYPaTkGoYHKS8+ZYTZwuv30e7CcmtBwMiorLdns/yrLE+4g2/Aa/wTDf8BKO5vhhVwD1E0TZ/aQUZJ0NKqNTMGmwo4BMoPL533nTEl36gQMW/76Icu3TDtbAH9ymCbDhvdekl4/RcvcmLHHBsIsN/PuM2iJEBb3ly//eicThoqPA5md1LkZD1jn5dHJ2lNJ2WbIJbwkL0ehVmVsGTVE5c5O+8uWRM5AuT/fJQGMhBjuM449acXf2K8q/7tE2a6c4rw9RG8gEuVosefH4cAFPpzVaSghLEft/0abwvu1JdwVODkHj9h3kcvt5OeKI+PW5USUA1gNMClOzaq5ZLhy/+LyIy5dX6vlS7hkV6tOyd1W4HMmVckueTcmONmfG3jP/lq4VzL8dWgY3cDpZIJ3S9dlzmufcfsabMsUBcVfmi2LIwYmLC0pezphQG28vhc957Ke1eTH3fbkR0RCgoChS9XEPQYI1tYqhISltQw1oTQtO3yOxg+YNo/uJ4tAHQUPHUP3UOS8gxNaByJbTUs5quH4K6fbM7BglIO42xCXQZYftwhWWZsRbslyzPk3ydwuwQLoVcYUPgBCwPtrcH6CE1l6N35fJvUsW4nqCzSFyKBxyiXFo/GLYQ0aPDsBfkyboQH4o869sxfa7ac8mo3vmbxvmZUq5lvY4D1k53Bb+wuRRR6m/YoefOHHauTV4iFB/3pNwo1pwkOUipt8ErZ5+63Og6py1NCgSzThHR+NzVSciIqqCp3+TsfU+YdIEJ9Ro0dcoZA/gcMz665/VC1R1PUWYLFi3/6Ks8+as/ikW10pQAiZqm7zTF1kNA10JmIG4KKANzDjhy2wFTpAvR5BaAWbCuY4mdWa/GhbiJeNcGCLL0UU9X5Vx6CCWC4tMtI3bQm4iYroxwpZLfLbbegkxcUyN0aD5GRCChtOQCwtyLl+3xS6DtAPehi0FGphy039BCzfRyPpJku3bE4e0BxpfTqAeu3sqtZ7fOJ8Em713WXxfAKbuZWXx6bkUQhbPyX+lrq3Gyf2Z6IJzd3TNcmdw9P8Ocb1cnTLI8SnQ1e1GZt9zrRZnOmaJ5/Kzge2EnLbvIlO0kDLJ6z2dhLa9LA+bnWwo7UMugN8AqIUJMEpXw38gzc9KhIWuW9OOKqAmJLyulxnLzDFhtsBtA0kprLUyaYFKWjNwC+0f8GDgvyeFraE45t9AZO3cxRkAw2de7bmoON5ROZi/oyWNvrn6QAW64z7xt/Cs0OTlqjJ1/KGXS8R2QNa2eZDorIHCHGYz3xZGyJfwE/zztraV7oMinIv/oYZA87mkz9CKHS3CMGVDgf2UWp5lSx3OFWkZS6SoBX3NsM2BMv7Q3/CSDCfearkz1KfZqKU3KJ5/3xuEJl4VKJ40NshNBbCqRauiNFkQanSNmOiAxk3iBDgs4Kg0Q3Xh7LYFMzqXuLsF6IauPzkLeVN3xbZmCT0DGiSkHlKCq/HTxKkyLp0KggNxYebHojNQlIJx9nZr4FLfAuIoB0dA6b2sPIVfyCA7UQ01zX5VCmRQH7BKLdy9QdoIgrVrSxUsaxj0LPZJnh1F+YLE90WEeXEbm+seO/SBFyHWvXsRHTL84G434UellF7ofTvLuPmg11VF8l8t6laQ87f9AxNgpwHSLRDkbhj1IMDmUlTVDe6Otb5K7EkbX2A7AN2/WBcSMOFIJsNjgmeRGN7nyHcU/rf0l/R2L/EwIJDIQftGqoHiC1V2kG//D7KC6VNneYrQhbQ9/8m+rJRhWiiPB+Ez/iNn/J6DiZxKqCOOl5Dmg9f1HaP2ZBsAv9E05649H/67aEPP5ZBgTZ1xeQE7tzFlwMa41QtC7aEl/1G+U5SWodoay0K3StRM0nvFfYDSYCYV+TbjZl/Su9mcEnSlZSsVN/fV5zySd7u8aImbraasJVVJ6hHnsswCpXD08Xcs7iSt8vjqNPFlFVw3FW6TQVyQBilBJsYGr7e4ESeYcrF69UgIYZonkz/MSgGA8o/A80fXHwxIEnkdbsjDh6qVvHYH1Ykm9KYG0uMFlHG3hE1L6hUS3bzp/UnM/jwnHirIAMrmd1FlMA/VKDDmFwBJ0SPq5+83FvB9WV9xHwDbbztlSlVPMzp/E7woJatAsxH9tovCZBz0KWZg8kRSGvWH+XvwmOYPLiomt6BcvC72B0GrstNWuPoa8aNh0UUrzrsZ+RgBkgMFHRyYSNIU5P9WKbstaDC3KpiQ0yYinjJ/UnUEOWnGrNHtnSUDr4tk6Wl8zPIeQTXWpYJKi0pJnqcaFqlbWqj1oujZgv0tFD1Gkhs35vxpwpj4BITTQoMk2RsFTmw5PDApol0xN3CnzfAn/xxC0/rZMiSGxT1AOyaopEV0s5SpFgiYiF/IENwICTzt8RxIwGMVPSyzUN4BfTURbSIvXsSvqOPSHQenjzqS0tyqgm6X0ah2UunBB4gs1lHbU3thZYhgQ7MtL1c91YnR6/5o632e4EPusMiATH5VUtgLOO/Xd3No4Bg41fPoTjgMmjYUI7EE1eYHITiYHlYBKyGi/cq0qJtGQOqsH7mleVq6hleDvLZPrRKa4nAGsEXQAGmwR9TxPqqOEj43WTBy5/IW6cYUTqE3w9vUNGTrstvOUXedHDtQA4fu2rjZWz2PLNquiX78vM1ixrWEky4C/KqbQf8coICrxtqoTZ7H3AAMQ/HbLitFaYTEXVXlF+lSEYWcfzuOOWdCLXSJex+zaMMJgtumZgucaivL+BCcxoCtBee84JIwCQ92xmB1isjCVA5/Ht55IrbZQYYRAcGeww7/Eaia3QzXXXwyqjza1ViqQgrjKRdtvtbkFx4tPn/esyi6Ci6bEX16Bxv8CYKNqEJVkA4zGTeUEwlaDbNczwE+hJHs0Naj+feJwWyAim7qrR85Rox/FUPdC3eU//6ssQcj/9op76CkvqCRGfoibvAwwKR9gtmlyav0excn0GE3csrik4/uB6IFk5Y1E6tMb8hwOFQgc4EddePdWWSAzp0gS4mawMHiFkR3bny4Ho0UfDF4AbooxYpiExSjOFlL1k8zocM7R1O9J+5ZZP8DgQ0xJ3vc2m/rV8lDbquCwbEMi4YqGSlucWVIo+7RhrrqZsyWcOtn9qnOqbGeeZubxofDTpeIPlqvqdSdCHG/zsYXn4r08GddEGwewrLxij9X1zv4HV9jrZnjf8367UNQYU5IUp7uk9SNPJmOsLTBYoUxR0oZDUYftKNZ/jybn41nlrTx+dj9W2nmb/JLdV/bgW9M6KeXeupnA1Kcyk3WQtzeikBnNmfoSB9cKn576p6PhS/sHEeGvh8p6FDsvKNDagRXg05PIriPIiMCCu3qymOIGtyqtLjVzsxUctl9F/HMf0YBYHEg6Db/VY91NxQpWDWF3OTW9fUeNc+3hISDVb54S/9pFwFAT8reru8aKe0ZSBH4TTjF9h+ya8asQIrK+M7aNa9hegPuCp8Q6kq9a0rVVqOD4z0ZNeIFYbuHbpyCSASxlH9oJJufJR7KNSWEKbMTDl+/+wmPVnMAPXR0Mp07GSYacF4NKsXE15KM9gPviJfSoTiych2/yLnMdLV/jBjjCnmkxhPKyrdhBZHYbHB/qiznCxuXYOuOLEON5BJvfoK9rDAmWiehQWiQibIokl3eEw60PNKwcGepCqydDRi8t0S4RTAxYYZlP4GEqvfbnJ6J4chdTneIM6NjaZMlPH/wxTVDGTBT9LXlfowY543DpmyoKxaECWGIK3VDY2uPLsv29NVHy3maZKEu7gkA+LxAi2sRvloenBr+KkVKZ80mCMSNkPiUdTNeV4qNAoWUAKnQbLzQlQ8c6seJAhSA6uKyewrA6A5mj9/KRR/d98sAlFYzOoQxgD", + "page_age": null, + "title": "AI SDK 5 - Vercel", + "type": "web_search_result", + "url": "https://vercel.com/blog/ai-sdk-5" + }, + { + "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDL6uuhOu9fjv2hASBRoMOLlxnR925FWIB+J6IjAE6DWq2R97NQqd95OfTRNMWR5peOaKEFjpdEFtvmwAERZzIjNUml2128CmlHStbVMqyg05YO1Utdj/VsV8Nc6FaspXDTG7MvyQftrh9butvzpTAL3YFCRbOaYuDlhvbp9fT07o3YqbQLiJU223vZ5q36TGwu6QnV2FlUQBH9O7/AYHmdX+wdhUCDHcJ5jB4g4hc61NjH2my08iV68Ttq+ORemKfy5Kz2I8kffvS7R6/i1ALS5o8gcn6T/oOtyH4vR2h/J3pkFVoTxBpG4rcAyZG2hGjL5zcDAce/VSIyjBfSOBMmrdToQNmz3CPNg7MqWGK65XQxYobSkH1ciE7ODGTSjK6hrQNX+E5Yqag2VKHQJHAmBBiHnv66wo+Y4P9BDyIIffxfcpxRc4VnAqSsIDURb6c09lWxnKBdzGzlljtkzE7UI8E5pYruyjzw91GwH0ev5OxhgHid8WEvqxl2FecyqUOjmuXlmRfu5My0rrNp8723TffajWFe3cwGAuaXH0bcDVq6O6z6vGLr+NlGDByC9upbiExY/hzHzKpsaVzRSfvTlO+LAJ4CwiYcAuNOrsJ8ApLCIe95DIbbviQkS/3mXB61kCv1TpWtu/bpzYtyETXfBJde/DGvdvcPFmRROJ46YgmHAg6rt5bxVpkQ8D1u7E7Cr09uvI1KFpFV8xCI6Sk3TVY7N4NPhpK/Mdyvue24rcuVWXGGRoFNGsuMq5tJ6MB47uVr689gTCOZO8aAkK6Idn+5Ld3NJnigA4nLIdTF49LXZyYyPNxCKHu4gqe/NcWuwwL0gSlrMR9fnJc9L1PR9uMkMbcYASGNT2Uwz9TApYdq1P2jtuTogABs9HPOOByCHFwWYofEUneLF+YWDk29eGomedM7RWRNqISsPz0cadjwPdtNORyf+ynNGZ/SN2xaH5oSHO+Uyeon2Aww1KaPHbjJjBMjXY+Kz8nw3aaSDjmcdiA1HqSqYpm9MO82c1omyNuL2/Z+a016B0VK4/uFvWIFVJLutQQr2tdHqUT3IimOPZaaO4KM33wNKsmGoJ1eByFaPZ4Vhe3fKrAbRDu8La35BeMS39qXAYWx6ujYu+rsaig+VdJc+L2E9kzZq/5JKPhkNlXCF5Rfkdnpi23QyMEqO6SUGzxBQ0g+dgQsjWCNCv5t/xhc3+TcrCgMUfnXSUDJ5+lbvZogPJ283P5qEETpOqnjuu8ewtacAjm0czjV6Bsv4K8Zl6PoHlJvRidcvZHMViWlZO1QKLAQN3q38qHCXSv9YMEhAUfQlCij59FelaMJwAg5JziS7ldEcGoF2cibz1E8QARl135SpU5MRJiEcZIpnXeCEkAWStc54E/h8YCSPI0boagPt10fiQ78p8Ajx+UMvr/OMdu8eCFESnRz1Bw/BVIPMvKI+3t1sn25uXSWps3OkgEYst92V/sikB4+C90BM+RqamGMI2NSnY9gZBRpJBb2YeQ94c3vnZjxrMj6WeeT1LgBLaGC6ygd1Y7xCi2kz6VLc5anjbDCDFjMw91pFNgnz700p2lnSaD+aQCkpPqAE6J50GEwmGiDTnBJ73wD5RVt93/kkZP+I/5GMdI4WfLLd4ivSKBmIcfQn5lIJuxuXIo38uJXocss3IVeT0h/oDs5AC6EkPSPTbdKlXrOg0z/G4JRYDITYRcwnfK4rS4JICjbjgzGnqewQaMULFju230NvNaCh8wtotouz8gLwgO7bvFfK8zacW57T8rMF8XFIGq/JwOXVn5elvjVz/a8GfDnGXI9bSzrxEYRpzvfAbIjcl3liRFvaK7fGIc6dEwSsYvA4rXNZNTR9H8eSyXPZGV8mVrRp+iUK2cxv+a8gDAoJ/NqxbSDzcuXRMLwxlmV5yIqy0Co0B+x/dYcxmAgO7Nd5NG8KdY9VrUdgT84jwLo7GxfzikCUv/tI95RImKjYsU+JvWY+D+IlXRgrbE9GW9KtOXgmyMbJ2HONm6O4rWgfJ1EcbW6ve5FhlDK1pKxZgRL6tuE7PeMrzMtptHMDWMs9PATuVXf5Kj5KaF0jiX/jVX8h9AvhQXfKGXGL4EZz8sIOY2bHFRBcFVDM2TPO89U3OJCL2kZYPDZJ7h/29kfm+rLBMnkrecheaytO+Ju8mpBAMXeD/BnuJ0PFK+WJeTlqmansnVxlz8O5PNmJH7/jU1EfweSn2j6YYANvVubl9CMcD3d/9ul0MaRzTSUCn+LCu6fdhP0OYOWOMMfn70JmNlSTUklINTdjueBSVikk91hAJ6r8UsRFm/geHyJfFZa1DOlKlAXuRz/WYNv8oUVeEt409fUdquN1tr7NUa0STxEx1G/A3xMfsnUougSJgqN/LBEBFxEFprQggb7D5YRYIgvE3LnvIl4pB3fRgw0I8GAM=", + "page_age": "3 weeks ago", + "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + "type": "web_search_result", + "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + }, + { + "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDLyGYrOjQBvP6DyudRoMuTwtMtXy71LcYbYnIjDXU4pquUb61gzt07c/MmJ39dkHGbD4Au3mUZmgn8ULSLQ+iJvi4hpWuAuqXe2C9vEqtRA7lSPx+/nm9rOiXr2Od9N2hu1gThoDzCsmUIvB78z+/y++hgeRgekOMRplPCOj/3eqyQ1OZVV9kwM5ZP6oVCWhU3IHUL40zdI63GAcEw+4qwXFAEqbypBG2y+CfFqNWTiUVH5/QyWRf+sMeMZDpPSvSmdUSFvNGbxqEGGCDMAKA7OsyA0ZqR/QXsxlGAkgcUiL39xSXtMBs1953MudcKuHTOySgtUPLc9HnkZOqeOy5kqNr0q03Vxklzd1tkwXp/PMvOh8SmhQYClMRw5V7fLiYiyWxb4gU/CSJRGV4CPrJz0oKHMYCz+jeuElZ/0UWQSE3RUAF0PcLi/FLSfklKWV94LtO2xbbP7clptICuX32d8KY0j+RUzfKenudoiMGtJfqHp1gmGtykXfnMWHefOZxtpQBp372lH/JQlVasi5uMHtD1dvY6jX+plPlfKg0z8wPDhjDC8Ehsf5wfTMU0RJGlsW6v8gEXvgFBsbDb/bmjtEDUqsN5UoCZ1FAvEgTDcDHCdX14zgJknqFhIlaS2HJoH/GDcYjnuZKVPePlUCCxL8H3D9mcqDq9wpAmcJDXJZwwh3TO9zD0vbTtVkSFmZc3i9wDmtTS8Vm+7tQn2/SstpYcSq0wkq94HIFXlkMeodiCxBZGb6gs0oQvxD9V497vkjhN6eMz5YPGbPEsNyIV/GYQf3dt3BdxFEbHKO5gcq5DkVnqgQmLS0XdISaEqBxIxITIiIyM1njpPoYJBvRDC0utlBIbj17BBDcAYOoi6X6l2/JL4hc2ZQiw/06LvqfmfPD4o5DiD5RhHMJezHrLgqyXjEavdhVr4DuTjsJn3FNmjo3GCdtkjjvsuXhoEdHTTGO15ECsYo/6MTMqRYUAaszXTidfU6ObMaw92t6c/LIGvqo8Niq+05wHQ3B4MmnB7bocHtjkh8h4ngKmdjDGPnYPo8riIsCA8jjloZmnRlF0KEz0a+1xnXXlb0n0qaK+eSZ8CH8hZttqDjWvTm48/YbxBOhiFX+Q1S80sF1d4o1BGs6OemRXrhQ8r9TjxwTIWMnvEoe4W2zo3i8aV/pS86aCzDlcePi7we5TlVY4knXxrZ4jg/vSiMr+QxuNxfSMKG3GB0WByzfhZV1A8M+15GpcxXHEJlG5lHuyvZ+PTJBZKoe8Z4flomc8I0H+q/GOa6m5ptthKvS2gtAvpwLwcwdg2KTMd7WHqDMKroO577+DrEjej5mKWx1slbTFRkz/bu3h6U4rtTXgfrnTbtBFXB1mZZ10HMl4vMZ7uAkgouv/dgxIowGjCzhHCqAoORP3MCwa/zFUgZkqMdvtIZ9/vz32ASeqoQLi3tKtMj/K/ez5hnKNqstE98FuYL6V2rhNC+7UWpMBmVgVz4EqntZrJjLGUdndHbGZ1gfgVoulVjUe+e/cbN8SE5ETT0ZQ0ZPHZUCxXmp+CBbGcfaJJzghBOXEYF6jtGOeeBUw0lxU/L4L9lj9uYyGb4TqD3cefWFcfXZS437QoTcgan1Er2PzVaIOVeqZHqtlRLZbCNXxzUhOOtjoS+6tcKOPu8bgCkouPNoXMU6xMKdw7kKsDty8QZMOXrWofjQx5PzB70Z0p0GhpiSwNAUZzp8Qwn01W9KVDa5KcU9BWEp1Lk5qudX1mfC5bru2SfsIacQ1xp12pwhe0iPadgTtR0i7Hpxo4LQ+Gqve7Ar1KDm1iuXH3gznz+reTPjqruSW3Vf2BUSr7TxKeG/3Mo5j/BG8rkq+J3aqROiRnVUntEbEer/+zsghZi8gr/O77iuUjubQnyxv0Ky4l/RH8nHpmAh0NPaZZUKGDjkLq5BxuMQrstpa7FhX+W/vzsXRcu+S0PXegE4HzIc80VPTJj+F35EtH5qRThqJH/cJcaJSp4L0JqZ5zCw16yS+sg8qqFeApZvcFBiYISh+DzI2nMwJgwY+TjTwdPWPhWsfC3HO2THBI446IVF3HTMREkFU+ERSz1eDFcPKHYyop0Af3FLogci/WAnVwTLSQoNgAJJj6tqsi1LerUmevvwFCP7aEfr+/nKMv9qHLOzj7CH9IErMxwcp3lVbQ8b8PAs1d+oHBMEL0W3NJipiA/WsHNSWGcpK14NUffxwx4dQZGiijD0YIKvFc6eQsrwdOi50auuc+AtrYYsm5PeQ0FbocYqzmjGqqS+zeB3XXs3COUlblHWRb/xhG6JKPSy8SMbTdrHcSOGYIHz0rwIaVaxFQ9Xf+GmA+NqQvh2KWZNRB9zrMkpnJC/zrRRDdEVr1qcpDvLgUeNgRa0GAsKZ7zXUVWcFoZoBwnGTPFgy0Vm3F51JjMHbJhiNbA1ooz5toPqqDenxRCTcSq3sg02aJ9kEOvq2tEJFZ1WOl6BuC6yrsUzbcVf0UWsYt97Qo2DeFQR0Mbnih5IlRPt8YFFBNK+E+orLB8EX/hqp2kqQmvif/75jQl73uwTiZzXpwewWgDYFmVsDgoPsOhNYViLPkXCumMW7QD74dwYZYPD8sn5FPHMoaqPqGWtEKt6Obf9selsJtfuuDkxn8Sr5lCzgJUhnjawQQa7zojtoNVsY7i+V4SMn+Aj4n/WvuXdn83+US8V7fO+R9LJK83vmuCIt7NG+vYeXovPXV+V1y/j/fQ4KxT15WOVVxt3pK5pbNnKz9KF/isPyFPoYuSkBi86iRe+NYNuiMoM1KAI3iiq6VDIGEq9MNvWyv5plCynMQoLMnLfojI39n2O902S+R6yUDud/uE7dkFixAWPe+XLtmwNw2Ekk63haqSB298CYMdZmIRiVmbnrvwenhE2BgKMyRBBM9sGAM=", + "page_age": "1 week ago", + "title": "ai - npm", + "type": "web_search_result", + "url": "https://www.npmjs.com/package/ai" + }, + { + "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDG2umNhfxWLLoy8/dxoMdD5PnOG5iue1z1pyIjBgsRxtJIke621D0LSSb5XKwrJyiEwaMc4S22/mcPM0QjCQ5snh6GATymqtv4py0M8qixeEEiNMxJ7UfbFWmN1i2n3scwxX0AMxku6FBUurr3Dfzh/W5nv/Vjok1BA3GPgc3i6Yp0xYDuQaMFJ5WOApQ+JnD5dBAv84qNPbKZD0BuW+62nPF8yD3mRs9fYLnA1J0NmMxCCw8aUF2J5vJjnmlIYb3vtsm3V4J2affLmgUcD1Yp/IGL+rGfbG0R1I5NWgJtsWglqYVHFl1VxVtYIDHbyWEed/R10MTKXCttY6Ew+mtpLbm9RDDzhkEsJ7Dl981G4uIFhfMe7FwQi9DbwQqB/TEKxNDbGE+Ba0UVEBIclhOCddVQxDwhkDbB7YPMTlZgeZgQXOSuV4sfj0OkLIR1tAb1+a+xQEB0a2e6npszXAwfMKRhI4zgBqhSbxwQbug1Qq0dt12S19oYMCkxOpsqpKn9fpB9durQhxtFEQ5WvI+RlH95ueDX2oLRv19kZ7uFc13poJhBZyRMiqu4LA9dDWZXyJi+sZYSSyUOhOuRtz4l6FKyW9Qp9P1yTVIqf803DC/7nhx5zHvPpRE0Fy4RQLO9BcY9Yr8LqK1lJeC8Ra6Wt2mqpY0bbufVj1upWJDIBbsUWzoPuq1Ar9LNuotdZq9HIZicSONp+VT/xJb35TE+9UKpr5KzcNh0BoWJcTb06v/TkSwp8QrS5iPQxKumNl61PcVVmtqn7bMYhTiBUT0byA0u9PtMaol5999UeQsqXzEin689lIPG/uvP7u2jUrUDuQRT1P1x4qm07DcaHRF5jOYQwemUmHdpsmvapyLO007uP9nkk5VomJFw3PRYfRLM7wLGKRlRmwVWCWLN7xOId1USTiRW6yYO9y2fqHfSUYLdKWjwK2eC8Pk+uwAX5rzf0yIKciVXZgbTb44Xzmy5286Yz4E/b2Lbp1BHrMHzS5Xy1ntBPCpArhq7SD+WbWjYaA8ycGm4yWSeKOtHNZV9cUSJoZR8+yHowcF8fsUT1bFp0aYnCh4YKhr8DiEHbOaOp2f92FXSEY1ZLLFj5TwO9q6bBXaMUQk68zSuBRhaePL17qX3A6ssxzPfDJ+xUzO1D8L/j7R9PQJkfp7qnQYWc/XPEn+KCjugNxFAk/AVRGp+FKGEHGJnKncxYy/UfeI+tb6RNITzJPiRYhZOPp1oUJMF1PUOWbdKQkePfhE8Z9C/wXGZFVa85V6FrrAD6zCIB6E4fj8Zjk2V0RqMJgFLR0BDHeEgzZ5909H90eNsXcx1KSXVogD5oYpM2UDnJkVAGzrk350q80Wntt+RCmGsr5g9g8MDCfuf32fMajx3e3S+qwzF9BqOm514eTV7XQnv6iqG0s33bG3hZi9lUCi0vNAHnoWaYCz3aKJl59hxIn6iDARtNRuH2UAQ7WeT/9CnkZoRPjDUFE1r2nP5cYApYYlD1x4JN08Bqe7pYzOkrFYhqSOLYkBMeKDmB3zXRRZcbsWo1oK6Wlw1Hx/VnVU6V2Ecq2GTh3TOi0jChLLR+pzWwiOG3DWTTBiR85hOmK4ljq7S5j79ElZ6Ae+BFSE0FJjJwsIGYOrg9X0YOyFjqQ3dyVWtzm3W2x4erWvoCwDDFU0J95EGLUCN9EAga7uGnD0rf6OCLyV1F26GmL7PB/zRjLweaWG7L+96aFnb3YtE8BQjxbuQHQZIoitfVN2qWeYbDNBtPpMlB+IaYARipqqWUYXCvY+1QkYke1vhBeICN+Cz/9ZDaynpOgqIddRZUdzgoUIkDJN200mQ7cbeZr13fj4XiEgQDw4remIICOYobO7TgqfBBRFiyeZZxRFtuqAdzOMQbFTk4lLBDmf1RMPFmZbOUDtRK/Wlbl8zaXEwyfYfzNY5rnMuQaII6U3p/6GCSgGQ2dzlc2rCrnjRDN7hbZhC5ggNCGNptJ37ZLdx0EZjiPUT4DbYNNW7pToWitok/P0Z7Ldi4sSk9szjoEIo8T9IkCMXOmeQ+9CnrqXaWRRJQ6ZSPUYOo0Y2w4w48+D+e9VEgdswsqVx6L/I7gyZz7CBTkfsg6UkcVdKSe6JFWbrMKQhQVkgpJO86A82TVMsWOMkm2XqW3XWaWZHU3HJ429lQHD6HThloOHV27bNOdxJFm1AHgGUeY/DwnqoncrVS7ZnGfpwfLw/LttxMQXp8m0dAsVFDXF7Phx4mQinsA3W5uDvWalZc4WQmGw0osA2RKQjvAuxJ99MO7ITTZeqLeLZ3wN0Kj3avcmMHgZ6WfFVpZ/FuZp5126d+Hwq9c7T+1vw/0PsxAgBOS/rFT5Me9Uw0gwhpL3CSEvhYOMnZYvs/V6iV7+jxMDWjDsnxkjkxC5FEl+HZNeMfPO9p+dl2JD7NBF1LtKn9cpmjeFBSqBCQk7rDV6MJIUN11rH2rxpPzelq/7KByeGdX/5d1aslY8xIuBqpc3TV7jhejEZN8hznUxOH1qoFP0jc48DD1mDOuySAdBCKdIWN0gjKCxr8z5CU6CmuEXwGSzaJ8S9eHOEoOKwU6mZvwj0VdhMHJxSIvT6QkjM2U/1LDirtBY7Ij7kT+xvogWSRjypIg+xTXsl5C0t5/KO07Edn3doD3KjkoA8txdQy5/2EUEuFXZd2LK7D4uX6+oA0iXmHM0xMBYIbWzMtPmXrJ3jaitbvgL9ChIkbtkVygSm38sJCUbXZhaYqnFxLZonLLP7TAvgoXxEgfTJ67n4MvhT5NTaYijKItGPU3vVBu1MkqbDMeXTPED/JWfeg02AguLg/ZwoPuqsKtuIaN1saR74UawIiX0rCDFfLpzzeTOJvw9DfgfowDHNs9P/Z4w3wlrIobDJpHfHTg5naSUTUyNYSmF7GnMVATwFoBtYem4JtKnnvgqUeQ1zkMUIGHv6/eKNkN249gQ10W2wbEFA0P1SNjm+E7IBZDo0+8Yg/NQuHpjdjmRb186sR/O7/Z+Cm0skbcTbm//8t3Zw2Fp5LY1HvHOVvnfvtcgCwJvCnkY0orZG7V1KcuBa0yl8fWoSjkluoITpFw1tm34msoJt6YZyJ2Wb9GL2Q+ElaydG1TGbI9541bjYjkGAbzvDrqIJjBQXkjvNskjY0XrgxFkVdy8yCbeqP/PUN+t5IuFnmqByWLXH03Zp8apNShNl9xZLGWw/PlEZF547S6vuQ7Z+pK6H4kBAb+4BKjXYdtwt3i1lfJbTLJ1ws2pxJ8WWrwLwEVH8dAVAQ/1NyhKeVHiExqZtaOwfB4IpCmjoTElUSeAAGhjUvShQTV8yzGajPgqSpMMraBuuHOSQyzHMCMlj2cqQyLiRMBsMgD2hJ2qGIYeXcsibJSBrtVZYKQ798e3mfGTCR6/ztEimurdi+ctNbUcVAri8437x/FA6yqZBBwPDyoYsdi+N/bcN+COFaryk5nWn6qd9J6NOXMfO5xmGWExbVLl5c++2iFcIQgHOyKTAXYFPB0X742uLeZkrdpwCxYr4bkYiwEzQ84/wn4QSXj+p6oijtb86g5FsnOeLb10BNe7SCD/xdT1OijIJmZK7diOd7ESs7GYo3ElGTwcyTEkxNpLE3MqtUKcogn2bJxite+L3DkUDturTzaPvVJb7+njiNp2dgkOmxx+ElnUD57iO3y+OqYcdg619kX2PG2DaBEK5nNFlaB2vqTLmAvRZHrS8An9/dy2eB+dhgwThgHoxyq0KyoZt/8m5IPmg7Zikdfy4kADOUHenR8m5MLwdlhkJAAjfZt23uXMw1L+XG2iKSx4jPsG3U6kDrF3S24lnAn3oflChUcTEeztocCc2JkREwa4HQqM10EdbFlprDjAOfHdsRTFpUIx+tASJJY3nlZToWdGkNVDHn02VclXdpuJAtNHoD5AFeL7B4iMpbSZ7Ddq/UuHq94uXc+pidZsxytF19sQzFxqkSlOXRelZhljS6pPU/bCcZizuk5k3fvJJtgY+EI8LTOiRZbWMT3vb7ATfu3Uo23+xavr+9hAmM9a2iC1xtY/PGFVjF8ttQYAw==", + "page_age": "1 week ago", + "title": "Google Cloud release notes | Google Cloud Documentation", + "type": "web_search_result", + "url": "https://docs.cloud.google.com/release-notes" + } + ], + "tool_use_id": "srvtoolu_01YQZbh8cGvpK2NTYcEos9aY", + "type": "web_search_tool_result" + }, + { + "citations": [ + { + "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCeDAdQWuM5Xl9vRsRoMAY1RgpqXY4nSMSnPIjBoPuTTCtOSZ4HsXYn0dsuOY8UkBVMQboGZ/AKhc/KnlJ769yI3mUTrJuurulBia3AqE767bZVGoPU4BPvfsD2wrDbjLxYYBA==", + "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + "type": "web_search_result_location", + "url": "https://releasebot.io/updates/vercel/vercel-ai" + } + ], + "text": "Vercel AI SDK released a patch update on May 13, 2026.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 256, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_choice": { + "name": "web_search", + "type": "tool" + }, + "tools": [ + { + "max_uses": 1, + "name": "web_search", + "type": "web_search_20250305" + } + ] + }, + "metrics": { + "completion_tokens": 75, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 16216, + "server_tool_use_web_fetch_requests": 0, + "server_tool_use_web_search_requests": 1, + "time_to_first_token": 0, + "tokens": 16291 + } + } + ], + "metadata": { + "operation": "server-tool-use", + "testRunId": "" + } + }, + { + "name": "anthropic-stream-thinking-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with the number only.", + "role": "user" + } + ], + "output": { + "content": [ + { + "signature": "", + "thinking": "The user is asking for the sum of 2+2, which is 4. They want only the number as a reply.", + "type": "thinking" + }, + { + "text": "4", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 2048, + "model": "claude-sonnet-4-5-20250929", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 1, + "thinking": { + "budget_tokens": 1024, + "type": "enabled" + } + }, + "metrics": { + "completion_tokens": 39, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 49, + "time_to_first_token": 0, + "tokens": 88 + } + } + ], + "metadata": { + "operation": "stream-thinking", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-create-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "BETA", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 24, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 5, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 18 + } + } + ], + "metadata": { + "operation": "beta-create", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-stream-operation", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "output": "1 one\n2 two\n3 three", + "metadata": { + "max_tokens": 32, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_details": null, + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 15, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "time_to_first_token": 0, + "tokens": 39 + } + } + ], + "metadata": { + "operation": "beta-stream", + "testRunId": "" + } + }, + { + "name": "anthropic-beta-tool-runner-operation", + "children": [ + { + "name": "anthropic.beta.messages.toolRunner", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 56, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 607, + "time_to_first_token": 0, + "tokens": 663 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [ + { + "name": "get_weather.lookup", + "children": [] + } + ], + "input": { + "location": "Paris, France" + }, + "output": "The weather in Paris, France is 18C and sunny.", + "metadata": { + "gen_ai.tool.name": "get_weather", + "provider": "anthropic" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01REdkoKf9Cn5nKu44AT439h", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } + } + ], + "input": [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + }, + "metrics": { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + } + ], + "metadata": { + "operation": "beta-tool-runner", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt new file mode 100644 index 000000000..5e594787c --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-tree.txt @@ -0,0 +1,775 @@ +span_tree: +└── anthropic-instrumentation-root [task] + metadata: { + "scenario": "anthropic-instrumentation", + "testRunId": "" + } + ├── anthropic-create-operation + │ metadata: { + │ "operation": "create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── anthropic-create-with-response-operation + │ metadata: { + │ "operation": "create-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly WITH_RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "WITH_RESPONSE", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 7, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 15, + │ "time_to_first_token": 0, + │ "tokens": 22 + │ } + ├── anthropic-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": [ + │ { + │ "text": "Describe the attached image in one short sentence.", + │ "type": "text" + │ }, + │ { + │ "source": { + │ "data": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ }, + │ "media_type": "image/png", + │ "type": "base64" + │ }, + │ "type": "image" + │ } + │ ], + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "A majestic sailing ship battles through towering waves and lightning-filled storm clouds on a turbulent sea.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 1389, + │ "time_to_first_token": 0, + │ "tokens": 1415 + │ } + ├── anthropic-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + ├── anthropic-stream-tool-operation + │ metadata: { + │ "operation": "stream-tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0, + │ "tool_choice": { + │ "disable_parallel_tool_use": true, + │ "name": "get_weather", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 26, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 687, + │ "time_to_first_token": 0, + │ "tokens": 713 + │ } + ├── anthropic-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 589, + │ "time_to_first_token": 0, + │ "tokens": 645 + │ } + ├── anthropic-server-tool-use-operation + │ metadata: { + │ "operation": "server-tool-use", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use web_search to find one recent AI SDK release headline and return one short sentence.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "id": "", + │ "input": { + │ "query": "AI SDK release 2026" + │ }, + │ "name": "web_search", + │ "type": "server_tool_use" + │ }, + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "content": [ + │ { + │ "encrypted_content": "EssJCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDJbsVr2UHzFeJ5hhpxoM3w9/4LAO3KSyZR0FIjA6GYJG7GThp7LNkiRYQItRnLnS1wy6GU3paY1h/NMJMX3SNbIR7AMtYv2IBfIaQ1sqzgjs4Hv+ZZwkFtM3T1g97VKd7F2mpPYgV3PtHB8SkrDpFbWmLFqHz5qVfJJQUJnOhTFfv6t3OGg89s5rQQonA90YyteQzaj7QV3EsTGAJKgNRbjtNby3IaKtBzH38NfoVFBCqeXWZy30qgqPljP2otYCNtOlCVvC0yi4DcsdZqoBMmvaozMiCKJVjYHzCbBtzIM0/7+Ro3hZc3VQQ67NqcTXnzHMdS+SEcNpoUu8od4psQwxXKjfUofgHoVsDlSft8FIOA1T4WlJBipoQgX2vfU02AK+cZ5ZdBAfVRYqewaqYVAgRTl5ylaXG8Kpwl+wFm2ZkEUbVentoJWJsAqFxW8xHFBAvl5obxVta182X6O4nocHWLnl5O+d6vgSL+ayc4hp8qiqPyVpPpmPFKtXeaopQGEfvp71LKNNFjOm3Jx2w+bPo6cbb446GnZgEook7XtJXp7Jty2nJV4zQeWbIvlwlT2BkPxmYGuTGhNFcvs4DuxRAVGOxsTEAk5zhNr9+9uqPPGKKmIwSkFF4jmGE8Va5otArw7nVT9Hu/mF148pkP1Q2JndOlZ8CaKMG8j853+jb6Oux/VocGVpJRdEfR7prBxh4A+ubi55MmSk7kGVNAwXVfCe1PSp9+p4iHuLzVz7h55QWqWLlWUS2UJectPiyPAa1QMonmFDi6pWv9ZBv2Iu+6yKX584oGxwWQN/fipp3ZYPltgOKPBsCFJXKmo6T+Q601uBG9cMIGzRgPl0pibytQ+ag91vjw2bgn9ASAeGE/MPla8BYsXqaYbk7lXyRffmA0cAChdRrAlRh7pgc8vlKUVLR0+hB+3znNm4/3g/RRiiQlNt4d4PeEbA9iEP2SSpZI5tYliJHbqEAhuU5wskpP3JIrTpi4skvIFZiSeM8CPF0HwbYJJx+kPgzKowyV9eFK5OfGwrVP1d/Br7N+5OxcYtqiRRxrBhnHZe/CrzAfr7lfAW945kduFMmb7s4UzW8xQly5gHh/AkoH/7vvogLCuiMGO3RH8awW3aBrSWEWWJm0iQXmTJBCT23zhHFRqB09PiYIh7JGHoFpHR6vkf09WybzwvULpFWzeeiqFgkLvpqPfhsX0DLAXCpgoNOgQ8woWxZ7m2B73uXt80t0RrlVCribOlM3c9PYoJWQvUAgQ1VHNJRyX+j3zsvSufzzpk1WOpuYf+lap2vbibDtRZ0h72Um0PQU8S3bHmcWRz1bhE+B+BLpVittnHblDr7B6ClCfaIZZvVPxgVvYhyCNsoaSro2Dqdb9UR48UIbp/wBJrsKMiFXdky+jLoUzdt2VqJ/dRXxQnPJnvjnDKrsuzaOxRXnoBcroQ0Z3n7h3JxiUJ8MEj9RzpiYfxocPzL3Ao71P02AgpExzTHkIgXGrSAGCpqO2jvst6C1hRe6kJWhQMl+tnJqmIFcmbOlu+WgmDfBGx9ZKr8DHlS+mgRXb7FORsf070qWQHsKy9GAM=", + │ "page_age": null, + │ "title": "Releases · vercel/ai", + │ "type": "web_search_result", + │ "url": "https://github.com/vercel/ai/releases" + │ }, + │ { + │ "encrypted_content": "Eo4aCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA2xoeQuNMkMTpYeFRoMn88FQvPps0GZ08VTIjAYn9VT5PJmc10NKT+kxwg7Y1NjAFzgkGnoz7Htd8IFlIySTKcGf+xMlPkmY+uUpiMqkRmCopQezkS54y4krvqDbuYq4j/lvKkz/OD7MYRFf3NaCXIUkc351WWWHg2pN04zLOiH2nAy2Hb0fFH8XnWngUr2kN2/nAnAEOEzJ3JqIfmf5yTgghjoEo1sAaQd4BlRa4vlmWiDs4q848v/6kjluTKrC+RG4vt9uPrCJDdqh11AMoZLi6eNbMMfJyQW+RUOSMsPSIUpPWKQKEFQeVkq7CbgNR7jeNHK4pNA8md0DV54TKrEzkCEdzTmvVPqdW3TZF+7C8mNKBfae4yy4uFFYAzyVIuv8+qriXgsfRGA5rnETQeqDZT/bIakCNX7zd1eIn6c0q1ffymi33VsRwPklPlK5I71HrfCZLXUKeeMLT7tpzDYL2n9jUeXgf9JWt8cOeW0YdRNnm+a1WdmfxV1JEoYHG+cCQ1jsgmz3S7ZxqS/ZekjeqM31WAdqvLf7xDasMLKq7mNSdjuB4ILEfG/RfOaU5WxkoA4YzgIteaqyNFIUPd91HXA8y9nWqNBE0o4W8Tf/rndxO5T8vferW6oERcnSU51SHKVVDoMjcCXbs1zuE6iyUIDJdl3vLGVL1gnT21V1yq1JkvAHaWOmxbfGXfVIAhPdXLocN7ZqWH7vXubGDy8FeomRLNyAKyHSn7FApgrYjH8avA6blhrK49CFc/d25IwGRNiMGmd0gD9aKLAocdwa8LPusRe1DIl4349eUYOTUHpwPHDTNNoZ/E+pZHNnlhVcsPH0IHewFLo11dmpCaLa/PQTr5AUyn0JRbyyImKhJEgxRHxMKs8lqQ3JuVd0lnoCx2ue1EzkRP+JqntseNo6H1zpXZ6W92J2yF1/iSbpOVmeKsmNgm5uC1c8YAU/HgkAnFG5YrFPYLdYKp426Gb2r25sKQGpi0XckcRxpqmUfPb4BU0YYGLCOQGNd0fr8jlK+rL+gwyanuTxBLlpCnPmCk2zy4yQ1kf+a/WjVKt7yxAc+3d+fDd/ijIPK9EuSjat+5G3/pG3ThUs7vYsbsXdAOcxrKO8WSJL4xBp39B/u7V7aGYmZFrzypEUUFWif391Bj8gGSLGTYu+qubIusI7ON4+LVO96zwUJdnmzwJxBb5sjCXWlz1dErNSfbqd0PMYoif6nT2z1VcnGCDgpT4QNGjpD+sS0X9fK5qa80mFwB81jwABGOuki1ZtRh4Bns66jwPAqLocbw7+5A3+OwKSYtolbKtg0V4CYOYKb1W3Ad7lOkFOJTKlkp2zY/2Sq/VWsGqGe0VkYWLI6AutQVtkVQYTizXasgMyvrND4Jl1Q5rXCYoM+2laUT8JFEDqyU2tPlQUlzVtAX7nrRawsUmzFAnic6dsoZDRvAHBx0EZClA3+Osz245Va9V9JMcgEJLIqWOmJBSljQOdjLEpFBimtipHeAq09W7Eh/srEA6XhWXKhCoGZuQVw2z/DJ9T2f3B30aOGczARlYu75AM0YdWvDHqzSza3oCnZkcIIB3svkcc1UPyUa5JZqluxy0Kk0cNQjFVhPcMumqQjMfN1EH+H7reskN459CE/HZr9T54KX3SMfrq5L2XivfsWlO4lMisZuITVLeikTK1jPD2dMG5gUxXWZXJYZPQsYWSEoltt6KtIwHTvT/WQcjSbpdTfluXibKq2DA40O/LCxCSUiJhnrzI9HsCsvPtiJHP56P39YGLmS20J2oKVKCwbgelyM09JB1EZySYmli6JW3WzoAs+i9u7cAyr5LYiV12LidFuHuIaGz12/v2YJ4dV5+oyjEgHv5qorSYmjH5kT4FfPQmnkYtxD9ZeYwoDnVONx8S5pmwyyQ1Skqk7YaIe0vC7L8wHEI/z3HR+cNNqDxKQo2pTAkIa+uoyfjiNe35es3+BXxFv7b7Tu6gYYLEHq2FYDDXOJiUpaAhk8QKEnwy5rW+CM7elGrbYxHP+tErUbHcuRSqHCvvgr9nTM+fMBGPoHzU7zzltAahqOCc6PQBIHDc3RrfPTOaIxW6F9zgWOQ1Ab5eI7TJWzqIiMYL5eqrunUruQBuWSLfA23uUs691dN/YZyediRzAS6Kirz9vPbHBRK972GxdKGhcwwsSywC5sShkrKdEkNrlEspLD85Afq5zeHpCh+hnhFDo3icw1MgFwBPPxFHsydIVFRPhDgEc0d7tlpS/kz7MUpR9cq7A289l1LRqeITnfHPXoqZ/FGMBS/yihni127nMV4IaWtN5jy6A3r1rccZR+Pg6RnWwhsTgyUiHMHav9TmeVyBKwNfCznw3mhrKvIzvND2rI7Oy0xViWSAhPYN8El5sjcpzkPoWJbCsGPmHE+ii2PxHHGqlvvWkIfTFECphntTBP9668PZFIihx3mRAsiORqrBLgdiQjP3TM4phbkjmTCx4Tle4C+KlhwUxJEmB6ojMdYuiLfUls/HnaFXDvW7QsJ4ApWWW5ohJGQfeg4f2/eYYnfIKVkSvt9HTtCoXXkZhXW56kVygDU9CCLhGtYbzs0eGhqz3hMlQwGRRlaOfveh5G+ypWEW4yfZH/rpzVq1CqvxihFe3GpzLbOEaEB8de8IbcGKT4zCkIvKbZsWfDpSHDPp38NM0b+4OfzlM3bIwTKVO/mnmso246ALl3EVA6Y2f6dsC1JxoQRqobM/adQIYNRKOlbRlD80gqNLTF/qPrsvbI5PjzSYgW224lRP4mnRJO9Z7MwQ6GUNbCjVb2VoTB6udE1hapxSJ8qIu407SDtIxN48/82OAZ1mqEjn78kzz6Tn0rxXIVJs2cUW7XKIcO7QIpRNsEvlHcoXK803+NFtZ979Me40rw5TYT4v9gn9DQC1jWXlVaYDkFnWhcdAWsI3JwFMr++Km/OEDFKjA7FftJdRl5nCusIu+teLJCb+CaGbpI/LH1AO3olxtO17BqYMrt8PYBK+1pzUK+kfRBzIbD+tPRgG9NSB2yjJZLNWSP7dUgSk5RVmUbJD0sobVX4w/rGwNKfIBSh2Y3kvEypEMb3EAC59sHl7VaoR0xO5c5t3Fe+T0vUj6T9ubnxC5bMeSE5DyQXwugVqQsLaUZtBbgpveRgHp6NCfhUivtvzjo007wERqxpBVGXTEjcZPFCVW0ig6jeqyMaCWSKY7dT71mczall5EVaFS8DTmKuLL+6SKE8FIi8chA9zcK9ihHrCxm9BNFb8/HgndHA0oflBCP2cU3d286Doz/m0PEKtlZz7o3hlf0waSErjrOjh5qXdVc0kFrUrsnl2Pg6dS1aPjAPHdsrrxIef6vw5gbZc6UXNrU4TTUbwMSnB4QT5fnCDGXMSuZvRMZmO69IH4cErGsM9KRb+/GmxBvUmbe0oc75ZZuOoGpqoVBRlS4G5LGN6VdeTlyZ50o1ka1h/wYFS3PcZ4ANHhQ3bx9t7fPqEZmmkXB1gEs3PC6soBqh9+NKtbB5lKnK7TX4qkczmAEWLBiL1PB0tTN9qrNAn6ouaw3zxw6pZXLWpJ+PRpkc6VwakJzUlI6jT8fs3gZpiB1M3LZanyiOM75v00CkUHfk/jKzVfpvO9dhhL9g4l5PSzdXnYSO5pF6wBpSmtH+ugRvVLyik++NZgvUGwhYUuT8O7eRxaGEcARF+EyUjV3IsE1PQ+gZGGtzSMVxr0rlP3n8UkckCZj4t/F2bS5wLYjlhmaSykYKLXvrf7dz8tZaX7gtKAS16uCM1gX1cRB2WOs0Qfwj7kIfp8/vX3x804RE6lIyD3bpYZYSP6pLJvH9iGxsHoYVIPgBJU8k6BzNiOKppZhB+bSAv+Aoy76RnQ8npUHiNmzi80ki6nhkKgJwm6QFS8ow09ZL5+oOCNBvWwfITFgAU5DKC3+fpc9bFWjaGEODBbnNLhkHod887nI+nR2iD/lPKFTlJg12aBdAWKWUHnbAPe5VNmRhfh7HFgNeuAKDHJAeEFe5tfmPEQBsD4POh1Zpui2mmSCvqSpppEArDxcUNbLPF7SYRlIHXYI1AEqfYbgV1Ijo6IMoWXRxTsUXTSQukW9XGuq1w0xiFgHfLiLDR5kStYaNnHjspdsukC1KCBm2rjqdanD68snvsvIA61QbdhtswHnQbE0bUVj7FCAJ57JMrd87KbipnnDuSmSaSPj1FYOSolyLXxUDMGeiiHsTXK2XMjW7TlLKWER5fihnqEmzPnV9gieFnOOgVTaj0iAfFraPEx8GsBBfrYEb6ZoVHNVn7k3ZuKMjUV8XRCqfhpGlyrYJFrC+aFM2McdM8ZvdPe8e8WtCQE3Xn4sWbb2DxfCuWHEoJzwjPi5HbrVAT0XDFHZOR48VzNjW//mUeCm/KmIg20llspdNvxjYGAM=", + │ "page_age": "2 days ago", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ }, + │ { + │ "encrypted_content": "EtAfCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOuHuK1JGxrX6+zclxoMWAc4pbopaBlNbkj0IjB0E0n5ijsZ/3q99pWnuVcXKV9fzZq5UrzvBnHK6qieyO1DYSEB+SttetNZP8ggYRcq0x6/Drd31D+BONbZ0MTmUMdFN8LEMNvJAoDDNEjqe5JSWdx9JyDb28InYj3ze7FhY7mmpTF3FFv6e5W4wSH2qlOxQd6f0DJsvsihlGaibpIeFIqAM3DWbmjOYPoD2HlDVVv3C8laNdbChUEwvvt90ISKxjFdPQIOn8pkxfEdzCE4Z1JdCkSXhslwIxd1x/FOpRhC3f7Bj8eILUzppnC0qgMo14VTgmGDZGJcgYLGZuWyk7gWdf4VMitLNyUyygV45eHMtlGcMBupT/HKXpKahj5YMXwpCGM1Ov4+eNaU0yfPzgl2cJDsqcQTq0KblfIPBuU4TSpnJ4mwChYVoHOwmeX1+DmqL1iS22Agp3Ycnqw0Qsms2b/95/RqS9x1V1kE3H9iu4VYsozEfI6nWS5qW4HStPNWKUb7r52gM7ZhD77cwpM/h3ol6459Y6ajjfZVTdrsbBqv6xvukERKjJTuB+jRwsqDTu8+tqJx3OajIW/V7dUFwXseOMlNG5YM7f3xBWRa9Dx0LaUete5c0blGGRqtIwfVX0czGFp12GK2Qned7M4nsEEzvHGr7LVXoc6sGr5hnjaYQ0o5O8hPn2hMhsWHeODTl5KiSt4dDlmGTjH5t3tIcNUqojzWuRRHGVjINTuhiEAsbD1yLfEUNIm+5G0oPu/luGa88knS+lWUO1ZZZIA5bzzlFkhmx7BRttYmsBDL/bjM8DGpa+poXY6i3eizRHMrQYgwCgBiE+uuBkzxJUJU6qg2p2fRw5fY/9ynhBXIvjIwxVSSo9w//S4NjDJeQZPuwwLXZh9BSgj8O+OhuPUYJf6n+yrjQlvvNE12VIqrBauJrMPmFisc8H4nHgYVJRpFyoaulLVIv33tj8ERzumigS+PiRBdJT8kPle0fPLkgpNZJ3BmBVzrHn1576U7yim43WkJ/oNDTqJSUxKEY7l403W+HL27Jr1Y6tt8jYk2m3+bDwpJnR7Hc6qvv3Z1fqzq+675Eb6I/h+bQp+mpDjBoxVYrgafiTA5bFeNNuQfloc2Hrz6QDdqbtXZ25eUrdGMg5bMClpYtaUM/83zJo+L4iTYCUcczzWy+ziTPNxd+DCqKjvEQuGmBk4SH3Wu7zLXsMMiW4cSAwMi4KyHgsiqV7gwZv6H22qFJ1TBUCC/kwFYo4sYHxkDLeamTzm1Vi1vwpiUG52ery8AcqQ3i/JwdwaJr3RTb0xwR/3jrBvaXzIrEzD+O/LBAZ30t4+3VVF9Dq/7CwLU5/10wZzZa1L+9xT3nXq5ce/mP0Ww/usXYVsPu88PGq5ho1cOc+Wp668RWz6mj9svhErboauWSCofNkdvHvD6fC50DbCRk+r1I7KCXaJPuoJg3hRuA+t7Vb1M913uIPVZx1rDKSFt8sDPTsz1rokpaD2r1Xpe23B8gWfFR4QnZ1shTKuzlBQon6kBWMqNozFLXLP5Zhmo6hts6ouN6HEE5qllKuUIFHg/S8xOgsGXvptXmimFXDcUz3T3oXRQ0Gn+OZN1xmx8LImeMn8fIRVCB4V0R8py8bFWyeFRqaQP3nWwnDdMXWcPn4bxOhWndjFXyi1wMMUog2DRtbFGepNGhcLQcnk9pZjns18FLZg95jnGCtBnjoylMRS2bGX10x8D+WTYFvjy2uSTeUob4JH/kaVw45UiJU9pa0hr3t6yyb9CHc+xFqzEuxA17+kPrw8b0rM+1Ou8dkNlPxMxGhJGyF7V3g4Vta1WB9cyE3Xl2oFPQMD/MfzG3ibsiCo8cSHT4QKzUocBQwQCfJmYf9fs55aX4Fo9kh7G8Ih8brK36PWFCet5sPCOKfC95DniP7PbUf51DsAWeebuWtWnrOOMtMEkwigol2WsstaDVi6WGgcB2xr7S5iAjKZRF9tG42VtIWMLFAWLn5DhJiLrpQA2KjGIvQUORROy9KVuz3HcRkVS9vf54H3Yo7EuePgQe91QkMvT2pi6Wq2Y2yWieBxYuUR/kuLujnwNJoZbTQMbWtkJ1wBW/sj7tMdS837mT0iTUedFkBQQgDk1umz80f8fP/JvkscQhRNmzXHbC0EsmAPxbBlBkA6LyeGKO4+T6qO+efwaDuZHlbo/+Ad4PTB7FgLqSNvMpPFtSiyK9i/vdrERNUuYmzL3FFvs3Yhm7g+Qtg57FrBvyEiDq/BWoYB0csm+KVvJzaQGmYt4EkDGfGzS7UHy9TnraykNS1wFJkMjlutZry7TIBTLqfqOBFGfQzzi6Nyg0skh6X2Bbom5tKQ5y5sqrBPMcZIJaabG951LRR60NYoVfQQhDR3pjmIupkEA2yC7PUIxjB0EAeRbnw5AxLx6HRhzNWL1j7SRY6N6LIC0Q/BdFb71mJrxo5C62xkNBbnxZ3jr9rrZLZy4L4sjvAB1Oa37oNzWl+JqoDR//jz8g7gfDwn5rqN9StlrTbYg2el/jG0a1N2wnX1zQPYQBHuc6SwRfF/Nis84T8grSKhXdkBD5Y3bueIySjw9BukUqxxUHbxsvuBTpg2Dg/Y7ECN2b+M7H9R2q0FlxLvR04AdsDJg9nZVdTDs5EFwnUBWclPLNiKtT1j93R81Msg6OBgsZiIRZtzPHOKq5YzUoYNbfhdiGzEfOZgRMj1mQd5YF7j6ps5B/XDqnSr7LRHCHZKwMmnoH94SIhfhztbsACzayiJsxNlkzxMM96c3w2gQmFi4kKJG5gmU4G1CmDA1vXcgrf9XYuhdBg66FkMYH1atthKUMz0w0+XwX18D8RnNPxa1JUtQ0DdZ5p0V9AtT2YI9RLGtwz6vw33RyQA6c6f5nwnm/f5qPCdLFLOexXTMcX9kXG7Dc6rE/RfBz4lZU70OsH2tHU9ERAt4SqDzNorDwtTikH6RcIVVO8vW9MGhuCMQRSID8lH+iZQBOFBgO3YUo5TzSl227Ua/pqewxY60+oq4g/8AOEYxqG3tXq61bCkWDgnMKR1EDco9+W56rir9RSnNHLjNF/2lOLWyY3zD7kwg+8lqA673oPiZiec6566aubqWJjSBm7GUJXlGQfqthsJpcHL2DRIsk21fvkmIjw+qqKr5XQVaPdJpD/TA0Iw4EyoUVER7Cnxz0uHYZi/h2rilARzx/eqOa94VUqnPMDqtGiE8HqWHtsoNjDfXA08SlKu8z5m0/FcNnYmWDrv5IPJxbKaketumP2ZW09N2EAUrsPyYJZpKCniG7wSqQmYLGcnP1JrOnMS9eh6XSN6AGu9WQzbg2RZU269W/hQjt+j9dIhN9RLvPK87JonKTgVPNUdCH+BYvZuJjPHH8AGZVQZuIlMIfTcKtW+eaGHa6+bQO0PRV62J4L8INcYryf9ivVi+L5HiLzrMpv67ZsCJxslgJpIlVJU1dJKhZoCW+kbOEd4KMu1Ucbbg+RvKLfRi43Dj+c2Kxf2P89mx+yukx8zIVNYUjwi17khBS6V3RX9oIyDxI/23s2SKomout5rFP5Jlt271FKHbs4E/W79zpFy3WIl+ydwKyvzaPXZePeH08ps9eWamiP0OppWSQIZXRtMOFklFc3WwBSWr0EYNbP1zOKonnHYMTWHPjXglBQHiqNJVO3UlCT7Ozz84sJWWwHTS3iOZwlaxdekwcFR4TKQcROuqtA53Mxy4LKhPFVgRvFcZpw2QcIDYeXWx1xl7ewJQ1DgzQDDKr/JwSs/tCvZmq5VKCAOeGXUuuRwa9i1l9bXAmTvBQZ0Fbz0DmY+s5BelpRoNM3SE7zxH6He3VX6Wk6rIOmUifupipjLnRpKQ3S1cQO3/npzxed9SjNONdc9MdzusqZ/7fLHBITAXDWRkfFFqDSSQnsf+61qNkyNDJY6yRMDeQQk6hPSuWv2AUZ9STGS7t9UlyqJbrnr+B6RMq1fFQ7g5F1mCaBaBWvUmxe0Ktou+G4LfmZRCoJm3ZFPj357S44XI+RBwOUHRxaBXSLSOVFkyHAhA2732W+GcrCMpgzNwjMVlXjN3ZegxO+oDUnrbKf1jc4+MAdYi2j4fPOsm0B+CvAAXJ2taqBX0JWkvC95BHrlz0AskMiR4mILPSCPTee+ibaEagDg7TUfgcioEGq6b8WFJRHgiHha2i8WOwNQNGI8RjzQK8BvEi8yL4qAfNihWLYlLVlc4W4tlUe7bXALICr9RCKhRPzgJWGGcitHPohkamEPI+Y9vlg9PJX/fQfDYZLqqQ309aljdb6xbo7kkjPwcDGeqbbuhRqMDO3M85mPQBQZehnHq+EJjDjABxyysuRKcBJsAiVIFYNJRMXxQmXvbGYyZSSd5WT2FYDsp25UkDeiyZK+nCR2FcqycAvsPp74eGi1oybgvNKZqXqR9+hu2wlZtv1HG67Z+eIAPhY5dfplQKaR07IS5ZQFsTUs0UpSrIZO5Ju4cbaG/3cpBZ+6wvnA9SC912vtAnz08618cSrCVdUb6IgvnXEf8itRtPWASqIQH95+B9hvDZLV6xI4T/7ZPHaDV1PMjHzVIs0UlvvTurZlwpcOJ/e+t3zUZEyr/y6bIU8m4MNx6Mmq5xSNQ+ZoGTi/hVTMWHfCvuJg1xwfxUsYdzhtH2oAXvXmAiGAfNLm61lvuSH8WXI26yyIiEEs62gPWkPcjKcTVKzvJfX7tg6fp2QDE6O9LJyzsLXC/RZDTapZYsTJovzZpJ5xltsphp5VfhGfDNOU0jr62ML0Y84A63ShNdj63uEFq4nqA+SQaI16qOZZzKEq4bpz3XkA2lZOz1xmmdH0u9eLwiCEGda3Uzk/yQsAmt4N1daR5ruc5R2YZ/+EoZ1RZhecv4Gl0IUL4yFhgpAl9tK+DOAsyKaRhu8geVHwKqw/8VQ/I3LlPLl+/JythWekewobOl7QHrDUXxVQlud8dCXi1HxbdSlVCj8Z2Q+2sxZ2jV5XVbkJwYo0cR6P8j5jBu65nh91CTgmm88fZsZXrYYro1oBJftUPJl1nBAgxXcp5WPsHmyuNWsBmhzuqEma8Kil6Kws07QTdWnw1SeWrj1f9dyfOa8VqaBljq4gYLXRnfnH67nmweZdwWDT8pHb5IAfhVUcbjze9nl4LQxcjehHH7rhvbcNc4X9vz2TeuaIcXoXO5F2IdkI62tI//DHM1ZZH2F1riU3Uz6GcTqA/giLQPukW4po/lBG+WerKJBsIyWyqknTvz8RaZ8Z+gLp9eKlv+UFgQpCDkbv+SUHcUG2MfwqjmgbH8pzpY9jxj+pQCvEheR6Ib/5powRQg9BRM6sGofOLKzVu33DkeRgD", + │ "page_age": null, + │ "title": "Release Notes | SAP Cloud SDK for AI", + │ "type": "web_search_result", + │ "url": "https://sap.github.io/ai-sdk/docs/js/release-notes" + │ }, + │ { + │ "encrypted_content": "EscVCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDAT+8zYgTM1JcJxJVhoMjw13uJThlc/AJrmKIjCyvetCNVP6qITUAN1WofuzWM4tdx5qWDkbqoyBhsOSuGj7k31lRepGz5O06t7L6KAqyhQ07e0yFeHDevO4JdKa9OfIcn0IvrM8c0dWooXhW2EN7mALE0Cs+t4jOhElXWRNJyKX/0OhZZcgusHOJhkCwR2bHnxwDLkVrWL514ZddnWpNu3bwA9pTakCqSfgWVVYzjJXNL6unIo7eeupGF8kD92e4Bv+HGB7TUVolPARWwrG0oG8FBhSP6Mm0INFlkPT6O3YE94W8ro2KsJlg39zxvXJq1kBCr8OBfvyz7Ck/+BRD/4oGeiFOUmSnmoq2tC2YDOflihYCCmfIh17uZgVeB09DPOxSK0N2UuAnGEs9VYmYpcszz4Uibj/7ibg2ejJNZR5Do7JBrA8OE7I49wwEDOXef5AY59GFDTG64eQ4UW3M3d24TBMFQu4mXYELDTRtkyO7WKiZ1eoWBs4whsouDBQu9YvxnxPsTajhR/S7NqhLmfpMWlIxmhoboEKty6AksPmAnFPuxsbXx3NuOpe4VGJzp0Xuaapt1HyWJjxGaBRH+VfrFldEbO2seQ32N/Q5FBPl3nfULF6/4EKqzVzyEu7GytzF/Q/FpU6E7AQxYh4G4DPbx9oUSJQ+khPFpttaROl0+0wkq5hrlEBHkAxr4qxH5ibUsvVjITHLPcIPyypGy+L6/NpS2w6jQon1ceG/s4/TeQHrR78ZvLcew3GFiNQjW8FnPzBeAPoGyY/ucrUSbvkcf8PNh8o52Z4KYudtFeva6UaH9GJeWAvXJz3vbcYrmcYts1HpQq6FbwoW2T10ie6ZphTCEa2VP8iTlYAYE+WRcEwFPPJDzoEwXlS3lbFicqxszHwRe7T5ltsYhZOLUYK3Kq6tmgM4kZ/CUN+qDJyHPTbkSUjZ45czNoZfZa5vwuMpKwCqvitwRwVknKDnlqteqlle2Fw/w1EeDWrcuYHgPvXeADQzyFAnJt0PVzWisvuwON7F2o3DqTyVTA+AZBm3TrWYXoOOZS7nzDpWU4hAwpOdwI8BSVY9QJmQVDosJ6Snuw7e+8hbKIR4VrUl8N/3SE45zl1l+TJykBOb5fFE8MocuGjanSPydfO9pE3gZKmpYR4GFgIyImCjM8KrennrqYVESUE1r5/FeDYUV/kq7FGtLNsmo0Sadu9Z9ERTuKY7hxrTc1QSbOBhW1IKs9LfqbfI804xVI5/wKJuikWwORm/Yfc6ptK9CJh2kDon4J3bfcPerfemmD3ZvIhbYnzsWu+qUdqGbH2CDVZkK38qNWqME3QOz94R2tAx+PVcD1FWtNstreGc6vvYKLHeeJ9YGJhFuSp0lik+ELdwI3g8k5PEJsmzL6ncwaAMTDaGcg7eG5VCayMO7bcaiAwpL9ayXsbzxN6aanmTk/P1Jyz8TMfXEjh/rswYlYs+gC5NRfxGrdOPD7Vrg3BiGHPT1BmvJUQIRoT5fM39RIWut6yHSi2jm86wPf/PPXIBPij4FrRrJHu/MXu2w2w2doH/ch70VyUa2JucrC1WcqZIL9sojEEdVmHrqRq8RqGXF/74BNvSNTdl/3BKyMqe54xRLLNMDAOm9KzrsB0P6FziCAaxc/TrMBy2lZxcfSMUcRioyqDDhU69doEBaLzINUulbbk7e74x1Q49pnnNs9pW1SGVifvh29wZmaC6+zRoKqCIjYjyTWAPxxBhaYubXTsyq5ykMKCNNAVQIbkGjvV/69isx7hasOq3HdDPg4avyWfllZiCtq6SQcFnQSQnFbZ+nppejOEfW+3N7dnsk+yRIkYChVEJuZo9dha0dL0lmSVWoghzFOGMyIWO+jnFl3P6OTRXImxha3xTLbE9y+EK6fMAtuzV5gBPdO58L2W0JNf0lOJpxpveyxC+KA7ic1qYjOQQJWXa78qNdYVQevQLqXBC5hDk6+Za50zXA2y9YYSUNZaDCiTR1E2yk7TYziXh9IIHwUHdcTxdc4uXeYMmNAmT9yh07FXbPvuA6xU0GtzMlONgjSK6JMfcwZhFdnZto507WsFcQ2Ff0qS+iwNbeIM0YNDifiFUHxK1+pOauz9+iWawWIISOal/jLSMht13r58trpTTDY00RqEMBJG7w0vLZNQuwc0qehzeQMpwaGZIAezpPflFGHIxNiORpSvE+pBQXjdU8ILXjas3cFTDClcFJGxTrpXS7JNnbsVVZElZ/nvVrzJ9D3i/IuWul2I0qKY2Rp7jdYZNrpES2RnIVKAwUwmCWuPafiW396GJqfrUTI0SQgrevI9jPH8LGYjiaCi4BmPrXK+YkbUbCWoyHcQdhtK9yHOcpmye7aaeRzznAcSghap/77Vr+nNEFbSgG5Nzad4nfHXtp+lPWs7gBi9zWSRdMyz6QSC5zbcwliUALVNxi4MUu0xdwsK6LCgvHn4VPd7CTZ1z8JlvOJ9UO8vgikqN02RZve7DTU97VnBBr8JWNaWSAGKV7aNp4nGSVT/YaWv4IJi3jGW1HE43M9+3CGkxvAoqmeBuL6wdTSAma42auo4FteSRgBgQhP5nhAamObhZJpwwzzxTVlSeohVyLDUXAYu1fqGznfvOvHXqwYHlpCVVKuPsAI0RQX0e18anYyUQ8NIezpvm6InOZzVfbBoTlFx/LNqJKxyQbLwi94PBB9Xw6ilkdCFG4SVrNsxpPao1XsE3vSW0fq9YLp6TN/klvLmMgyfZXeYhxGuMujQJ4WeRhBfw27JEDVcJSe99ifCu+QY3cw0rN9vrmkuWi2LvqLb5D7PFSsqXFaKqvJtuANKdeP27iQaxU3EepRy7K+5QyM/AN0oSivKW2WNxq94mh7zC4OlFZPOeblV1SLeae2Vjs4Mp4Xka2TYhJuKEHzBiOUaMrn5tMc1oikhBLOXgPzN3H41eL428k5JuFTvrPLOq0aaW6LrpYiUxQSjMyAR4LsSteT0A+WpOzjK2ihG0x/cWMjnpPB8ccdoykR3Ko47zZTtE+t9uL514vRsDjmkrSQm0dst2sB2dhAppoN7SX3Rz3SW1IaV9xugtXjtUW4b3onbGRwuN6hsVa/QxGfT38bhqbRhlR1UWk6xvnH8jJY/OVFTsPAjpdGDIsL/7d7mz0uPvl8xQXTicYPdaSjBiVCbvVwtVUKjRRJrjUGtSIK1s4Z8pVRA4Q7g0ASizLlAhK646prOPQZSwIhLTmPsufQy36SAp1js1xVHw8SE8jEo3Or8p2wbCv2XurHuN+k102z/ZMkSfBUacvA1YuEoyrpmmKgYvoAHorkFXeySZF+ezBCPCVC4FcJdBiAe+zEt5mWGvV46kKNMa6AeuRhqxTPXzUp9pZT/RQ0c9FE6PTHaPWfz9NJaN/GYToEBslHu1Os0A8Ta/G2QdzGzkY2bZ7qZ40mUwkys2U2HANPto4oAEybh2Xke1RKvAJOFvtZ14w9+VWvk5NX/9uvx9ppaOSpwpijCs8j3ONhDxYAoi/JcaXWwebDP0FAWHnBRcCXNNQcdV/VbUx+FDKJ+iAjvxrDG1uftVkWM0b9+4sDRThetEZwc+fU4Qsu6bsNIFz8l6vr891caMzlSBsZ972OwBxfruv0YAw==", + │ "page_age": "1 month ago", + │ "title": "The next evolution of the Agents SDK | OpenAI", + │ "type": "web_search_result", + │ "url": "https://openai.com/index/the-next-evolution-of-the-agents-sdk/" + │ }, + │ { + │ "encrypted_content": "EugeCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDFfN+Gdu0tEt3bG04hoMJrX0vSvbN7vJPrwuIjAgnlcb/vmr78XYUzBoLVYPe76T0mzspJkHE0lu3KegnPXNBAsqhEa7sLoJVDL3aI4q6x30FBB7NfnSu5x/6I03cvAsqIovNQXY71xbXJrf3Kyh0TAaIazN51YlNCNBYPBlCpoLnYjRPhq9S7ooT9sHbBoXWiYItBRX/0I53s+9tEDiDyqUhB8UYdf/NqCLJFI1aMapL17UP+p3QkCU+fRLtELQFhudeabA7MyFWs/FWImBqe7r/Az9dNtqXvjFU7k+aBwkD4E67k7M5BP1+AvyiymWvEhU8VdawKHxg8HC8/i6t9NbXM9lo56WjWr8ZGcfhbQt5xeZFdT6aaY9xSWJACnF+JOXRXwxmPRsF/jougyx/EpVX4CM7+gLiNOLYe8/WFK9TIgG621PU7SOM7hhLk5JSuVdRgWvxq/S3BFwmSC7xzWpFyszPlKQjhWXcgxQ6Mu4N/9JubWWAG23nDZvCsWjxUU1S1HLSFGxkUsvDMcNkRK36m7trEiRtjFqtoaAyV+rT/ihtwB9kXiCKQza74JppLoO6Qzu5XE/aIcjeX7dCbPPBCLjfwkRdo9x5M8gn7DeGXtAS3+BIkFFRnB+E1fFcTMqRnIkxjD27UiMopupxN3xUTLYVrYLtnbJ2MGW8XpCnSR/GXjuUlooUZmw16EarVUNn5Tv9AanCgURICiD2qviCrVAmsYxTLieA4CzsrQ/hxYrlf7bVJhS9YJCuqOpoLWsmhLQE5LKc2iq5RsqryUJYOSOVGHnCQSDAi4zpq6E0PD/or6wvlZe2l8yP9x3r7u2aUUOVdveNY8yzA5aZ/xZf+w5FitrITRt1qMFSF8d+IKi8EG1w3Ez41oolOj66ggKtmOY8s0JhjCQJsTyje+dlylJAlGZSpn/KVtyBZNiCIwyOwz6j+KuUS1sAU6dIwRmgpOntPoytgwLzoJcCM7f0USL5HMGPOQfrkH93qjs8OhDu1eN5kszSfHAgKto3nz17kSZRh/P4jEvcVnbA1OuiihjHmm+OeaaVEWFRb/3VNiDHDSaoct7MTt8JjFzA20nj6d5oUFBjlXXyejqBSgXE5gKtWYefl+uGMDUEXGA4VFOW+DxR8E7iqgqlOFmw0t2t7qCpidTmdR7i4dOPV3SOUZ16CzALW2+CoxA72cFFqpSNNPm3r0Vmk0GvBlEDN09KjR8t/0O3of8fi1949CBcKcs14uTdaZ3oiIhZKFMD+MeIPLO8r4ta5zsaCPpOKUYGzFLCXJ0N4BaG1X9tzxF+vTxDwqCWpsVwSuHXy8u4XsgyDGIx2+J9KiDvgPcaI6Ss3I+FstXWhPNaJlOFA9XItk2r1ZbdUFSAVP1cbastnjg3QVnsPQeudmIeMe5PNdeGQPSnMHCeK+UQuTeZZKB2zKivcS/LueiTw2hCyZBnN7PqgeMxpuv+Fd02M5iSdYTTAQgeI/jQ1wSVfvNTxpD/c9xu7TmIjnlXQl7Alv4C3QJW6R/2ZMjwMY8XCRBDfaujGBeaNiE8ddskUK/uxV+bJtTRSHquoJ/W/oN3orbxBSHQ7hdTwYpfhRG1zzQ/HyswgFsqd/Z6zGE221qfYOnzsJoAzhxYYKIxWY5PrGRWQcNEd8PxGdz5dqluXt0dKgPW8urRI5MbftA1Erk1ZduLn/wRub8P3L5Xhyisgzi+tqSgkHeWfLCEleP3gcF+ugbpBnzSUjc8CWWh0YrPv2d6dtfvOhJWcgLGDhPbgVRB7LqaiFaa9k8C/tFsGD9hf/ni0D9b7ddc+iWav5Z7jRa6i3IbEKmE7Hg1ULfT+gkC+U/xKmKY32zrxH1wysPK5/mbcPVyli7vFi601fMnN49J9Ynb8xPFbcNZdrR6q7QZMcGkiFajDSuNha/QxAZLdXrrOOagvw/MwgcijVSQ3APD20BYko8PFdUP1HfErmwdB9+SP2XiwzrB3J/PMF3BLYSElLVY4Z91HAt8Z6KGS+yqgXyQ2bIfjLuaRek/FpX+ffj0vV6lR98cJvIE44cQNbixITrqVFQkuh3zGc5bWNMTnva/i02JEaoS2l7zfGVi32Xpoc8d61a/Iyww6sZHUl/BTv1Cj/1/h0aBBjEt2F/eE9+7bPme/6FYTomEStuYqz90IEnLTdGJq8hbZQmviGYGdSruHKcTCs6pfVMBaEvY0JrHwhNLtz6a4ZaamR8Lf5CxTT/T0AoTtZG5iBs43F93kDgyw5l1d2Key0tFffQPNwnxakJyPWKzdUWiSUQC7hpO0xB22n72ktpSgUdJ1MFx9O/muLPwBigUCBI/QaGyvreh67mjj+9ZKhbGltaN7gX/IEdHgyuDRI+XkYoViENJsT8iUVIzSy4KRWdbWED9Bk5WsKwlWt6/WvkbtdQyuNLXqsnlTIuMYNNQj3UYUiwqcNmisaG4TWJ8zziLZi0hy7jHHl3J220A6j7PZ8Wu6Xffb5Ii8m86g4oXaEkk+E01T7CJWY3avWHOEJzZ13W+HzVEXDz0w6exueG6XqYe1rdnd/TxMXZXjNV6nfo/dKQq58T/LdLuqFXOGvMFkwGzxZoo8wsddCy+hL7ZyINnVWpFlI2/M+yyKSbBfLVfVwuA7t2wdUcBV22MQMAQPOGE3iQXm43tCXKIocyxOWwGkzzMYGNojwwfJSg5GibnDiEWZEix9zBiP1P79WkIsOvqdCe4RHy9mOWau7SYz1POEjU7ndP70rejh5lp8vpYMdE971eoiub4G3Q3O6L+WiaOmOQQQzO59GeDHOF/AUnp72zzR4O/6pNp4nAcMPkyazFucJ8kW43n9zUlxob8HPrztjFok92uBKDvb31ZPw6ifZd3KxCXMv7fZCFKG+K5/+jCNYWx+atPr1s7+6A5ZatnwyiN53ZYYIHytglCzLzMJGcej8h71LY81YJBsBUidM612K/OBU9jYZfYt8X/pxzB/U9EJbHeCZ4tqsVlVmKvaXPhxM02IKuE0wkj/h3Xrvavy2g66XqU+HGG1GxNGLGFnI4MeNQORJ23ASamA+r3gqiF3u6/HYkt+rpcMCZtWUZ/IgN3GUEUU6NVIO7qPQcypYc7uURWWtesUzKWfyV7rUdDth/sD+EmIio0KB7VRmDWuMT4zIRNx6zCEpQzZ5KbuKIm4HD+ACI6BYjSkdVPXdNbOTk//xZ8Kg80CCF+7e2jBIyx6KpLTsCmDmH6nQ2+mIp8IS91d3/vrSHyzzK1Hm282NXahSWZWEm1aCkD8H4DBWhKeBUze15o9j0DWEFCPvI2hX0lSYcOjmp0azGa2sIiTw6bPmbKgDTvZqH1kHqTfDQRPGIPW2hV6sItomYECFcpKVaRDZ5e2M9N/FU/cojrFymLJLGbnkXK80Zxi6VqHbx+lNVChT8pr1E8xINldjOiPvOb6OmipEnMdSRv9UlryGY1WvyI4olfm2QaVLSooo5+FTpzNijqOwF/PWoSjF/IHtjmM+SrtrTdi7CgrLG1RcQnwIIMH7t7WgjtiEVjVOpFN/S8Gk+Rcc9RykkFzas3LyVc99H9LsgPlg+eGa36MfTrISJcFjxcV+QV6/+CGGJHePpX2wDSdwVn28ajuhAVDT/WGiTIZpBYHUdeIcku73KWSf52VxqTzG7FbTlg4Kow16qwAvprNIIxGtWSaM1Lnif2yDpXJXYB2SwBUFNDMhbhPJk/aGoPgOdmtxBb6Qd9YkwgeK3I8QLfS+yngwJzFJEJSMTBD0HAsm8TSY37blTrzeJFeJ2ayRTka8PVvlWlqmX55ClgSQIyBpO2QSbiUJSE0SGh7j6Q2vpP86u5qYdSKp6evEhSdfJRbdin4eW+Bxh1uP09u+At80T/hlbrF88K64+46KZL+zZn39NsVNVL6T6TzOOPwiMDW4TAugax66wLdQcEw+bSXz7C4CFH24WGGbszFJEyHLGrkrMZrkuvppTUgkyoS6wuhkO16I2GC2a2iVLS1gCw6zFAZEKg0kEIUYaOtqmD1H1zhHZkDcvV4twKR9AQ15q2gfAkRiWeCzV6fVzyEutGWXt8Pgzq36gxJADlgGG9ePOyUci/V3jlXv1xJ/KeP0jWKmrp1sOgNl4aqHFtYqJK05ijcX3j6rSgMwop8KJ59RWPsiKChmgQKj3D+KNZFsyXTrYayj5TszQY5geo5KHjGOObWvWTwkw1uFPNKDnGb9k+NHiXp3/radd3objg/oSZ01ZnUWB3jer5dvBDEcl8buFqcZzElaeJ7dvA1QrCU4i8C1N9Px43BnX8P1bMKgVFwfXeeWq4ukciTFFakCnfJsQb640enUCHgCzjAx7cBBzdILzMiaeYDlEF9k8cX6SQCYiSPaftKbB9sZoxHFaka16t0gSMX0szSIbDUpS7F6S33cM1M5GIQi8sHNWptDlsYTmXIu5zqx0Wm8MlZq1EXncNakI5WGiy3HLpz2EwVUq/k5inQAJLYQEQVPOlioB0T0fNkqxj71sDgidaiFgE/SaRw7UmofhYPtJR7jG2yuGumoYUOJSk/99N3+CzqbqqfFHsI8E50XY6oz/hPqOYMD2mpYdkcNAJkufOJhLU7+1UHdPHaq1YAH53sgodpui2Tm3SRlqo5z3EoCjsBP+hGtcIN/sgcQ8kp2MKk2xk74JDN7qTxGImf6dAIyVsTUf9caNNk0ezdgOKb0U4YxmZy4QwTCqss27Drw6Y6sOlp2ZaOMyBKNv246917qvlI3t38FmGdqATnUhLnbP+ziP/To1kz8jaJo+9wIY1l0L4Iy2mBnUj8k51XqG+5ROad66S73nNaanKhpslB9MsOKSsYJYjlHpsKYKEKgLWd8qsmPD81jvW9wSV29uyy38I8a+xZOfEmEwM6itlykZOfo4xPuwjUs6WWIaJ+9FaULBoBlDMrDhK2JC+7bCsJTelWAN4Dc/6KVBF1IDzkHqYqRkWCOJiv3+bKlN70oTcE5Jg2trf4jIYStNzrl8xACvXqD2KCy1MtdxiBaH41maAIdtsfA0eMUF1wLCaTrAd2x4NQC/5a4WgXbZ8vODc5a7rwLfxYnP/SF3wlMNf7OEIt491cFQLQe5zALfUpEiMB2ELzIvn5fkNpNvspFJVu6f7omwIgnbmknFS8hmf9+P2rWVXXU2zKy44yrEpO4c1Q1hdXqp69TtYZ4Y69warYQlSzOW/MtnCt9WNGOuTj8YAw==", + │ "page_age": "April 8, 2026", + │ "title": "Tether Launches QVAC SDK as the AI Universal Building Block that Runs, Trains, and Evolves Intelligence Across any Device and Platform - Tether.io", + │ "type": "web_search_result", + │ "url": "https://tether.io/news/tether-launches-qvac-sdk-as-the-ai-universal-building-block-that-runs-trains-and-evolves-intelligence-across-any-device-and-platform/" + │ }, + │ { + │ "encrypted_content": "EoQSCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDOBJUjRPHm8Bi6Qj5BoMMq4p91nQCL2rfO/LIjA6rWYsI/IwH8NSPfxEhRHF+I7KHbbCxRQldIEjo2QwdTn3+jgu6sefh6CaoeIB3/8qhxHUirJJNUcgoTvHuzO6MN6DzAFcUCUa/sHJWNXRkM4aPPMm4rIY2PMkKJz+e/FqjQUu/Z7VwNAmY3jK4GeoiPmxyMLoVmcX2R59BfSkePt/kIYiPUh1T/KLyBx4vRuzDjS0vfxnwOpkG1XEuW/duF1mY1aQiQuARNNoiM+bKVGa+Ukg8voZoenyWB0ZbfDnSDwU/B8jyvtM/74cW7NpjhEK/2RVx8xSV0KtRbPPpUXhuu5vAYLoCSZ5a92iM5LVg25cup2APVPpI1pQ4zN+hmlU5H5aIrl50qzYl/xoFhfLm0io+pAOb79fM3AfExfeOJU6kaZ2m97lmuWh2C+acYdz+4kUGxceNB5cUFS2NBXvxTusRUXCL1VP5laPOHyeO9uv9LE3QXZeoTGZFwlL1Bopr9cKHfhTSXOAj47CHegNMs+T2UYnauUbMM4c/8FS9tnM9/yRDtzZkgAVthMS64nzpekbAdbpFUBATrRombX6uSmTewSzsBgXU7sOPBJ/L7UHdHR0CemNJuowMWW3k2NqQAjf9FKzTtrnhUppsSwC0MRq8cpZLGBNOqmJ3xO4TYdt4nZN+Cpu5+OD3YyNSHZ/HQRs/sMZlX+Anh1cVBXv8fhVb4X1AcHfpn3AQuq636nSbRo12JseQKh/sjB0PQb4+f0tIz0ivC6yxayx3AfI3Co6Q66Jm9WXxmHNw2C0pvXX7jRCoJPjedv6INHfk+9/iNP97qhQ7EQh7rEKbtprpNVXN6liJW9oKafVik5+c2NbFIgDX8FDXIEgyUCrF4HOKqFELwRJ3+0TJsYJjy4HlyXku/cNwPBwqFG+dCMjUBt33m6Z3ZzZqMzz1cs4TwQQucm+eNaq/QyfZ6tRpuff4KVDIbrW5gTkGkKm9fVK/QvA8pbmyvZWQnvSZqmP6AgQy9My88aZBdrXrbETe4crvrOj0cVSrVbq8TBrIEgBGbIUzDEEthxWFAmKmare94Z/F035RUyx0WKOpET+B0cWzFKuf+HDFNvCYLtoVSUQarCM+oIC2fBURpURGnmo75PLqFC3liQtrS1NeBUo4sAgGt1BlqaHkbVJBchl2NIMgjB0ElZd0PiJRQD0s9Ga8RcJgF4Nytd7K7EWRnnSj1CYMJKEyFW6B/3AcjaCE0n12H6Nf30amZImSgQ1LeRDB/aiXCBDhwnuak0nhkrPeH6Zp1jqxaKQYfyXvnyXZ+5w3Jqx+/2+3jKmsWpuDrDZ5GiFbheP7cwxq9dcGjE/0QL9USJUCZPMV/H4tdWXqXsWof2pzqXhIUgh6BsDp3d2Ss6HbqYTddMjem2Yc43+qxjecFosq+5DB3Bqae6PzAVihu1WCcl/otfs9D9meIHpMbAS8D73ypaAi64tzeRg/busWl2tgtfAiroyEkScIbfmcvezaizDosXpIFnCquqHL+ZcP5CBvMe3/6viIzJhJ3nFB5VL+Tcm/JI2bA+SFJI4F6CeSP85xHuFcxrb0EoU172qO3IH2kUTXV8GlP08C+664AOOVDKOi91UqUwsEIYrOceNARLjYssUhPPWqx/7AOi9BrxfEfpVbyG5HPm/paeUK+2oCEoJHyGPCaF4V2EcZRlXghz35dOYrQ3SjaRPWBEzZCg1nbBTJpEgTaVJyjyu42LODcFh6pPqoI191KZiTyZFyuoOhf8DHA7P/mFy6YF/sPYsYu7JJn/e1mcxCwzqz4D10FHeqCvVAUGGyorAE5w1vamiOSrvne4Z4gWAtXcANTyNuKQABpv08kcPe3k9tXKK8N2pUiRvLJJeBUmCOrxBHfo6KpQG9MOT/q5bM7lqEEFNw+GrslkPufldA0CMccwxmxTC3WO+gcag0nbhwf3o4RwDLVLg4buEEcKtQd8p7zjOeuf0raLf/Tn2+QRnnGAzrqbYUboWJfM0AqjLRdG7wzc98QRK1nXuLLT9p0csPbd2k7IwIUh3Nn7QTKZURri3i1tlXduhtJ5Dl65Rk3XvOSIk9g89rnLswEXtpMHfuxFRtQWLO+tzyd8Lb85KDw+zelwZgCcVBmfCjFUOm+5Akd88Q3+0/F9pusmWGhQWZdLTt6D80pBwU2nsDxc8W55tyq58oda0+JkPb0CtJ79jC2o/DDx7UdI+o78s3Yj4dmMGFtpo2Qzb3LRSejUNlgrN+EO9MRGFzUcoCFM+mJ0h3gq/87eaJDtrp6xIhkILwwuxLwcqiZOmevCOL9eUTjX7s2srTLZZr1MVmprOisxUXT5xqfJSbfvjKBhpeTcULEqUe3IkadEgCiEuVVTZNjVqzbBn3E33bIC3Wl7uHQPB+ImVErxojW6XiI+gKeqqbeDrU+r2K+8IIBBq4aR6dWW/qALy14ERQY3BQw1EpwjdTe8blF2FZ8aXR6qI76zwzg34pn2szhJFTlP5kfBmP0nhkGVqCyLHnBpgwYH1q4UA8vKe/5TpJ4sD+DD578dTZz/M+dsHqPH+W/cAg8iqGxT+St3fXYR8ReMOFws1H6s0JrbUi2uL8U43oW8/GYjLfPieN1G0c/B1nzt1qAVVnrJUjgu8et3AjfjOvC3Z0RqJ+cB1WmJ5XJ6ZzZZ2IJaIkCbbbDMnSjVlgcoZzp07E519VTOYj3Aokd2+/9iOjllh/Zud8znp8sz5lJZskGCdwizRxAje/EGqnk7A9/91JzL382hKegG+yFyJfZJVFJf5JfMQlCW21IK2Gd09hyLsEnP9rqNZ/b60WR+ztfrum3mpHckXvzcJUzzGBgY1E5iPEyWiA2Y1sxp9dorHpmz2K/sO7EVSs1tfLlEHfokHhOc55Xu7N/o9U1CraQ8nd0O9vtGUuBFJDX/1PMwx61+KzaPT6T8zRRRFLrMgMyVFOe93tGQrNJZ1MD0AYl8YljNHpKj9A03D85sdU3G+mYo0YMq/KaYWBxgD", + │ "page_age": "4 days ago", + │ "title": "NexArt Launches Complete Verifiable Execution Infrastructure for AI Systems", + │ "type": "web_search_result", + │ "url": "https://www.globenewswire.com/news-release/2026/05/11/3292224/0/en/NexArt-Launches-Complete-Verifiable-Execution-Infrastructure-for-AI-Systems.html" + │ }, + │ { + │ "encrypted_content": "EoohCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDA3wUbunuwEMie71hRoMPfaxkZfHjC9ioMCZIjAleOM0fUbI8y59mErH7so1QdEUGUKXzR4MfG26gwYU2vFInP53877vR3TkVmVZMAMqjSDSqZNaCk3grt2x0NB4EMxyJ51MbOMHZPGacIrqqPm2sg5ki2noyUEIJlVwSDMF4tiQXFOxIr3EeuOe9h6NnIKDYTCInTmNRIAHryTxHq46cxy+Y6IuTZn94cD1OCoqyP7yTpzpKq2pCfSu82aBXq5QvukXaXz1AJDN8PEz6506Aw9x6S5DrZg4mfn3cUoDQv+IFzyBQ3IS6aiRr0LBmVli1hBhbgKbGh6FcAoBzHpupqBFi3CTqiUndMHJ4JuMXVzEVyE2VbKTBkiUqJ02q+Xf24YFndywCoP08yxLnLE8X6xs9Sq7NdZSkEBbSbP/5fBK0VY+30Txm2f3ppv2rKv8ROIxAAuuLg3tZ/m02EBz0WUt9qCGTOQza3sNoOe3tvyFOPyc6Q0YFIceurrotcbZHY16CgcmlfF73ItmymMUwhGtmJ+f5DShB6ETW/ukA7qbECOs2hWZWHfuBsUR4LJoNEvznNhihlLW2luLkIEpWwWX+eeNBeVFHj1W4ZyLDxfI6tvs88sVKynxP6qN2ToQDxvySN4Muvqso8ntDKfdOq/GYwhWaxFW/yg8tN0+JJSmGhrNx+40PfaM58EAYYU6rPgdi0ut3gwKdJziAlS86FOzFwIoyl2DehboRlDp7/HpuKKSqb/Kv6mMkwQBVG3CYysiLBAaR43tCaMYdZKW78PRS5Wc5KDkrPwUVbY+TCdw2urVw5WHCYk9owUb61IeFoFV4QAJttgW+mm/KcokHkI///P/pCji1jm9FPAvBPZRKWTK4Qp+dxedVRCH52LcXeHk+mYlOKEqXd3Ska1FfAXSj/7FUu05XJt2Qlk+q/xhZwmpKlJCHqR1EKoUoV+WRMg53CQs1k8Rr7RxxXdaEyds8gXDttROMbNnYHth6Ks4rB5akVHqVXHydePAvoicduEbPS+AO/EMfZpzqrjcxiC2Ac1vCpmA0eI4bFnlh+qJltus1aXRJ+tp4N6QvpY7vdxC3uOPZS5nsX/CzvRN/WyN4qX3JYZ2ImkDDvOF2SZldUesHh7i1+0v0XEhwY0U7F0WSBFXFfXakodfN25sZ4KZQqRJsnyFUp15TGyHpzh8KPyFSuALPbhHSPZvptlDYiJjDaJGoY8ZnAQaRA2peJi2lfZlMjysPtvXf1VyMk9E79pVSTJylnR9XUsugc6Ik9cabB72Pcx4wR/b6Vy2OjW2ZTCk7LhSdz2SFyvTl58ka38JME2iyTaP9b/tBUEFZ9lSAhpj0krlyqdt/QzFJqIQX7ajK6L8p6u5ucReKMVy/LY0l3jHFJwesO68CKz1/G/gs3Swxcpnz3tZsoPeBbuWiLj10X1f9kqb7WXrzbd1Zhp8ce+ffdqKcj1O/MUoywb/3lL5Lh4WxnbO2nun0k8n2AHaeaihUDq6jbk5YwysY58HbZevL3ePnXJqx5hj7EShEcGWUUIe3PJtPouvmzfNwzP3LgcAVpwYIMxNPNTxkTLx5CBcSsZWVE7lAOk9Adt23aRDBbinjsSoZhvgnJofRGLPSAgfOPYpmzg+JH8WRLsA84jXwD3K832NOyuJdrAg24G3WsuwMe/ePHTR0sV9hS8dpGF4SkMCwraw3FPc/pSCDYqefseuzfWYPaTkGoYHKS8+ZYTZwuv30e7CcmtBwMiorLdns/yrLE+4g2/Aa/wTDf8BKO5vhhVwD1E0TZ/aQUZJ0NKqNTMGmwo4BMoPL533nTEl36gQMW/76Icu3TDtbAH9ymCbDhvdekl4/RcvcmLHHBsIsN/PuM2iJEBb3ly//eicThoqPA5md1LkZD1jn5dHJ2lNJ2WbIJbwkL0ehVmVsGTVE5c5O+8uWRM5AuT/fJQGMhBjuM449acXf2K8q/7tE2a6c4rw9RG8gEuVosefH4cAFPpzVaSghLEft/0abwvu1JdwVODkHj9h3kcvt5OeKI+PW5USUA1gNMClOzaq5ZLhy/+LyIy5dX6vlS7hkV6tOyd1W4HMmVckueTcmONmfG3jP/lq4VzL8dWgY3cDpZIJ3S9dlzmufcfsabMsUBcVfmi2LIwYmLC0pezphQG28vhc957Ke1eTH3fbkR0RCgoChS9XEPQYI1tYqhISltQw1oTQtO3yOxg+YNo/uJ4tAHQUPHUP3UOS8gxNaByJbTUs5quH4K6fbM7BglIO42xCXQZYftwhWWZsRbslyzPk3ydwuwQLoVcYUPgBCwPtrcH6CE1l6N35fJvUsW4nqCzSFyKBxyiXFo/GLYQ0aPDsBfkyboQH4o869sxfa7ac8mo3vmbxvmZUq5lvY4D1k53Bb+wuRRR6m/YoefOHHauTV4iFB/3pNwo1pwkOUipt8ErZ5+63Og6py1NCgSzThHR+NzVSciIqqCp3+TsfU+YdIEJ9Ro0dcoZA/gcMz665/VC1R1PUWYLFi3/6Ks8+as/ikW10pQAiZqm7zTF1kNA10JmIG4KKANzDjhy2wFTpAvR5BaAWbCuY4mdWa/GhbiJeNcGCLL0UU9X5Vx6CCWC4tMtI3bQm4iYroxwpZLfLbbegkxcUyN0aD5GRCChtOQCwtyLl+3xS6DtAPehi0FGphy039BCzfRyPpJku3bE4e0BxpfTqAeu3sqtZ7fOJ8Em713WXxfAKbuZWXx6bkUQhbPyX+lrq3Gyf2Z6IJzd3TNcmdw9P8Ocb1cnTLI8SnQ1e1GZt9zrRZnOmaJ5/Kzge2EnLbvIlO0kDLJ6z2dhLa9LA+bnWwo7UMugN8AqIUJMEpXw38gzc9KhIWuW9OOKqAmJLyulxnLzDFhtsBtA0kprLUyaYFKWjNwC+0f8GDgvyeFraE45t9AZO3cxRkAw2de7bmoON5ROZi/oyWNvrn6QAW64z7xt/Cs0OTlqjJ1/KGXS8R2QNa2eZDorIHCHGYz3xZGyJfwE/zztraV7oMinIv/oYZA87mkz9CKHS3CMGVDgf2UWp5lSx3OFWkZS6SoBX3NsM2BMv7Q3/CSDCfearkz1KfZqKU3KJ5/3xuEJl4VKJ40NshNBbCqRauiNFkQanSNmOiAxk3iBDgs4Kg0Q3Xh7LYFMzqXuLsF6IauPzkLeVN3xbZmCT0DGiSkHlKCq/HTxKkyLp0KggNxYebHojNQlIJx9nZr4FLfAuIoB0dA6b2sPIVfyCA7UQ01zX5VCmRQH7BKLdy9QdoIgrVrSxUsaxj0LPZJnh1F+YLE90WEeXEbm+seO/SBFyHWvXsRHTL84G434UellF7ofTvLuPmg11VF8l8t6laQ87f9AxNgpwHSLRDkbhj1IMDmUlTVDe6Otb5K7EkbX2A7AN2/WBcSMOFIJsNjgmeRGN7nyHcU/rf0l/R2L/EwIJDIQftGqoHiC1V2kG//D7KC6VNneYrQhbQ9/8m+rJRhWiiPB+Ez/iNn/J6DiZxKqCOOl5Dmg9f1HaP2ZBsAv9E05649H/67aEPP5ZBgTZ1xeQE7tzFlwMa41QtC7aEl/1G+U5SWodoay0K3StRM0nvFfYDSYCYV+TbjZl/Su9mcEnSlZSsVN/fV5zySd7u8aImbraasJVVJ6hHnsswCpXD08Xcs7iSt8vjqNPFlFVw3FW6TQVyQBilBJsYGr7e4ESeYcrF69UgIYZonkz/MSgGA8o/A80fXHwxIEnkdbsjDh6qVvHYH1Ykm9KYG0uMFlHG3hE1L6hUS3bzp/UnM/jwnHirIAMrmd1FlMA/VKDDmFwBJ0SPq5+83FvB9WV9xHwDbbztlSlVPMzp/E7woJatAsxH9tovCZBz0KWZg8kRSGvWH+XvwmOYPLiomt6BcvC72B0GrstNWuPoa8aNh0UUrzrsZ+RgBkgMFHRyYSNIU5P9WKbstaDC3KpiQ0yYinjJ/UnUEOWnGrNHtnSUDr4tk6Wl8zPIeQTXWpYJKi0pJnqcaFqlbWqj1oujZgv0tFD1Gkhs35vxpwpj4BITTQoMk2RsFTmw5PDApol0xN3CnzfAn/xxC0/rZMiSGxT1AOyaopEV0s5SpFgiYiF/IENwICTzt8RxIwGMVPSyzUN4BfTURbSIvXsSvqOPSHQenjzqS0tyqgm6X0ah2UunBB4gs1lHbU3thZYhgQ7MtL1c91YnR6/5o632e4EPusMiATH5VUtgLOO/Xd3No4Bg41fPoTjgMmjYUI7EE1eYHITiYHlYBKyGi/cq0qJtGQOqsH7mleVq6hleDvLZPrRKa4nAGsEXQAGmwR9TxPqqOEj43WTBy5/IW6cYUTqE3w9vUNGTrstvOUXedHDtQA4fu2rjZWz2PLNquiX78vM1ixrWEky4C/KqbQf8coICrxtqoTZ7H3AAMQ/HbLitFaYTEXVXlF+lSEYWcfzuOOWdCLXSJex+zaMMJgtumZgucaivL+BCcxoCtBee84JIwCQ92xmB1isjCVA5/Ht55IrbZQYYRAcGeww7/Eaia3QzXXXwyqjza1ViqQgrjKRdtvtbkFx4tPn/esyi6Ci6bEX16Bxv8CYKNqEJVkA4zGTeUEwlaDbNczwE+hJHs0Naj+feJwWyAim7qrR85Rox/FUPdC3eU//6ssQcj/9op76CkvqCRGfoibvAwwKR9gtmlyav0excn0GE3csrik4/uB6IFk5Y1E6tMb8hwOFQgc4EddePdWWSAzp0gS4mawMHiFkR3bny4Ho0UfDF4AbooxYpiExSjOFlL1k8zocM7R1O9J+5ZZP8DgQ0xJ3vc2m/rV8lDbquCwbEMi4YqGSlucWVIo+7RhrrqZsyWcOtn9qnOqbGeeZubxofDTpeIPlqvqdSdCHG/zsYXn4r08GddEGwewrLxij9X1zv4HV9jrZnjf8367UNQYU5IUp7uk9SNPJmOsLTBYoUxR0oZDUYftKNZ/jybn41nlrTx+dj9W2nmb/JLdV/bgW9M6KeXeupnA1Kcyk3WQtzeikBnNmfoSB9cKn576p6PhS/sHEeGvh8p6FDsvKNDagRXg05PIriPIiMCCu3qymOIGtyqtLjVzsxUctl9F/HMf0YBYHEg6Db/VY91NxQpWDWF3OTW9fUeNc+3hISDVb54S/9pFwFAT8reru8aKe0ZSBH4TTjF9h+ya8asQIrK+M7aNa9hegPuCp8Q6kq9a0rVVqOD4z0ZNeIFYbuHbpyCSASxlH9oJJufJR7KNSWEKbMTDl+/+wmPVnMAPXR0Mp07GSYacF4NKsXE15KM9gPviJfSoTiych2/yLnMdLV/jBjjCnmkxhPKyrdhBZHYbHB/qiznCxuXYOuOLEON5BJvfoK9rDAmWiehQWiQibIokl3eEw60PNKwcGepCqydDRi8t0S4RTAxYYZlP4GEqvfbnJ6J4chdTneIM6NjaZMlPH/wxTVDGTBT9LXlfowY543DpmyoKxaECWGIK3VDY2uPLsv29NVHy3maZKEu7gkA+LxAi2sRvloenBr+KkVKZ80mCMSNkPiUdTNeV4qNAoWUAKnQbLzQlQ8c6seJAhSA6uKyewrA6A5mj9/KRR/d98sAlFYzOoQxgD", + │ "page_age": null, + │ "title": "AI SDK 5 - Vercel", + │ "type": "web_search_result", + │ "url": "https://vercel.com/blog/ai-sdk-5" + │ }, + │ { + │ "encrypted_content": "EscOCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDL6uuhOu9fjv2hASBRoMOLlxnR925FWIB+J6IjAE6DWq2R97NQqd95OfTRNMWR5peOaKEFjpdEFtvmwAERZzIjNUml2128CmlHStbVMqyg05YO1Utdj/VsV8Nc6FaspXDTG7MvyQftrh9butvzpTAL3YFCRbOaYuDlhvbp9fT07o3YqbQLiJU223vZ5q36TGwu6QnV2FlUQBH9O7/AYHmdX+wdhUCDHcJ5jB4g4hc61NjH2my08iV68Ttq+ORemKfy5Kz2I8kffvS7R6/i1ALS5o8gcn6T/oOtyH4vR2h/J3pkFVoTxBpG4rcAyZG2hGjL5zcDAce/VSIyjBfSOBMmrdToQNmz3CPNg7MqWGK65XQxYobSkH1ciE7ODGTSjK6hrQNX+E5Yqag2VKHQJHAmBBiHnv66wo+Y4P9BDyIIffxfcpxRc4VnAqSsIDURb6c09lWxnKBdzGzlljtkzE7UI8E5pYruyjzw91GwH0ev5OxhgHid8WEvqxl2FecyqUOjmuXlmRfu5My0rrNp8723TffajWFe3cwGAuaXH0bcDVq6O6z6vGLr+NlGDByC9upbiExY/hzHzKpsaVzRSfvTlO+LAJ4CwiYcAuNOrsJ8ApLCIe95DIbbviQkS/3mXB61kCv1TpWtu/bpzYtyETXfBJde/DGvdvcPFmRROJ46YgmHAg6rt5bxVpkQ8D1u7E7Cr09uvI1KFpFV8xCI6Sk3TVY7N4NPhpK/Mdyvue24rcuVWXGGRoFNGsuMq5tJ6MB47uVr689gTCOZO8aAkK6Idn+5Ld3NJnigA4nLIdTF49LXZyYyPNxCKHu4gqe/NcWuwwL0gSlrMR9fnJc9L1PR9uMkMbcYASGNT2Uwz9TApYdq1P2jtuTogABs9HPOOByCHFwWYofEUneLF+YWDk29eGomedM7RWRNqISsPz0cadjwPdtNORyf+ynNGZ/SN2xaH5oSHO+Uyeon2Aww1KaPHbjJjBMjXY+Kz8nw3aaSDjmcdiA1HqSqYpm9MO82c1omyNuL2/Z+a016B0VK4/uFvWIFVJLutQQr2tdHqUT3IimOPZaaO4KM33wNKsmGoJ1eByFaPZ4Vhe3fKrAbRDu8La35BeMS39qXAYWx6ujYu+rsaig+VdJc+L2E9kzZq/5JKPhkNlXCF5Rfkdnpi23QyMEqO6SUGzxBQ0g+dgQsjWCNCv5t/xhc3+TcrCgMUfnXSUDJ5+lbvZogPJ283P5qEETpOqnjuu8ewtacAjm0czjV6Bsv4K8Zl6PoHlJvRidcvZHMViWlZO1QKLAQN3q38qHCXSv9YMEhAUfQlCij59FelaMJwAg5JziS7ldEcGoF2cibz1E8QARl135SpU5MRJiEcZIpnXeCEkAWStc54E/h8YCSPI0boagPt10fiQ78p8Ajx+UMvr/OMdu8eCFESnRz1Bw/BVIPMvKI+3t1sn25uXSWps3OkgEYst92V/sikB4+C90BM+RqamGMI2NSnY9gZBRpJBb2YeQ94c3vnZjxrMj6WeeT1LgBLaGC6ygd1Y7xCi2kz6VLc5anjbDCDFjMw91pFNgnz700p2lnSaD+aQCkpPqAE6J50GEwmGiDTnBJ73wD5RVt93/kkZP+I/5GMdI4WfLLd4ivSKBmIcfQn5lIJuxuXIo38uJXocss3IVeT0h/oDs5AC6EkPSPTbdKlXrOg0z/G4JRYDITYRcwnfK4rS4JICjbjgzGnqewQaMULFju230NvNaCh8wtotouz8gLwgO7bvFfK8zacW57T8rMF8XFIGq/JwOXVn5elvjVz/a8GfDnGXI9bSzrxEYRpzvfAbIjcl3liRFvaK7fGIc6dEwSsYvA4rXNZNTR9H8eSyXPZGV8mVrRp+iUK2cxv+a8gDAoJ/NqxbSDzcuXRMLwxlmV5yIqy0Co0B+x/dYcxmAgO7Nd5NG8KdY9VrUdgT84jwLo7GxfzikCUv/tI95RImKjYsU+JvWY+D+IlXRgrbE9GW9KtOXgmyMbJ2HONm6O4rWgfJ1EcbW6ve5FhlDK1pKxZgRL6tuE7PeMrzMtptHMDWMs9PATuVXf5Kj5KaF0jiX/jVX8h9AvhQXfKGXGL4EZz8sIOY2bHFRBcFVDM2TPO89U3OJCL2kZYPDZJ7h/29kfm+rLBMnkrecheaytO+Ju8mpBAMXeD/BnuJ0PFK+WJeTlqmansnVxlz8O5PNmJH7/jU1EfweSn2j6YYANvVubl9CMcD3d/9ul0MaRzTSUCn+LCu6fdhP0OYOWOMMfn70JmNlSTUklINTdjueBSVikk91hAJ6r8UsRFm/geHyJfFZa1DOlKlAXuRz/WYNv8oUVeEt409fUdquN1tr7NUa0STxEx1G/A3xMfsnUougSJgqN/LBEBFxEFprQggb7D5YRYIgvE3LnvIl4pB3fRgw0I8GAM=", + │ "page_age": "3 weeks ago", + │ "title": "Azure SDK Release (April 2026) - Azure SDK Blog", + │ "type": "web_search_result", + │ "url": "https://devblogs.microsoft.com/azure-sdk/azure-sdk-release-april-2026/" + │ }, + │ { + │ "encrypted_content": "ErIRCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDLyGYrOjQBvP6DyudRoMuTwtMtXy71LcYbYnIjDXU4pquUb61gzt07c/MmJ39dkHGbD4Au3mUZmgn8ULSLQ+iJvi4hpWuAuqXe2C9vEqtRA7lSPx+/nm9rOiXr2Od9N2hu1gThoDzCsmUIvB78z+/y++hgeRgekOMRplPCOj/3eqyQ1OZVV9kwM5ZP6oVCWhU3IHUL40zdI63GAcEw+4qwXFAEqbypBG2y+CfFqNWTiUVH5/QyWRf+sMeMZDpPSvSmdUSFvNGbxqEGGCDMAKA7OsyA0ZqR/QXsxlGAkgcUiL39xSXtMBs1953MudcKuHTOySgtUPLc9HnkZOqeOy5kqNr0q03Vxklzd1tkwXp/PMvOh8SmhQYClMRw5V7fLiYiyWxb4gU/CSJRGV4CPrJz0oKHMYCz+jeuElZ/0UWQSE3RUAF0PcLi/FLSfklKWV94LtO2xbbP7clptICuX32d8KY0j+RUzfKenudoiMGtJfqHp1gmGtykXfnMWHefOZxtpQBp372lH/JQlVasi5uMHtD1dvY6jX+plPlfKg0z8wPDhjDC8Ehsf5wfTMU0RJGlsW6v8gEXvgFBsbDb/bmjtEDUqsN5UoCZ1FAvEgTDcDHCdX14zgJknqFhIlaS2HJoH/GDcYjnuZKVPePlUCCxL8H3D9mcqDq9wpAmcJDXJZwwh3TO9zD0vbTtVkSFmZc3i9wDmtTS8Vm+7tQn2/SstpYcSq0wkq94HIFXlkMeodiCxBZGb6gs0oQvxD9V497vkjhN6eMz5YPGbPEsNyIV/GYQf3dt3BdxFEbHKO5gcq5DkVnqgQmLS0XdISaEqBxIxITIiIyM1njpPoYJBvRDC0utlBIbj17BBDcAYOoi6X6l2/JL4hc2ZQiw/06LvqfmfPD4o5DiD5RhHMJezHrLgqyXjEavdhVr4DuTjsJn3FNmjo3GCdtkjjvsuXhoEdHTTGO15ECsYo/6MTMqRYUAaszXTidfU6ObMaw92t6c/LIGvqo8Niq+05wHQ3B4MmnB7bocHtjkh8h4ngKmdjDGPnYPo8riIsCA8jjloZmnRlF0KEz0a+1xnXXlb0n0qaK+eSZ8CH8hZttqDjWvTm48/YbxBOhiFX+Q1S80sF1d4o1BGs6OemRXrhQ8r9TjxwTIWMnvEoe4W2zo3i8aV/pS86aCzDlcePi7we5TlVY4knXxrZ4jg/vSiMr+QxuNxfSMKG3GB0WByzfhZV1A8M+15GpcxXHEJlG5lHuyvZ+PTJBZKoe8Z4flomc8I0H+q/GOa6m5ptthKvS2gtAvpwLwcwdg2KTMd7WHqDMKroO577+DrEjej5mKWx1slbTFRkz/bu3h6U4rtTXgfrnTbtBFXB1mZZ10HMl4vMZ7uAkgouv/dgxIowGjCzhHCqAoORP3MCwa/zFUgZkqMdvtIZ9/vz32ASeqoQLi3tKtMj/K/ez5hnKNqstE98FuYL6V2rhNC+7UWpMBmVgVz4EqntZrJjLGUdndHbGZ1gfgVoulVjUe+e/cbN8SE5ETT0ZQ0ZPHZUCxXmp+CBbGcfaJJzghBOXEYF6jtGOeeBUw0lxU/L4L9lj9uYyGb4TqD3cefWFcfXZS437QoTcgan1Er2PzVaIOVeqZHqtlRLZbCNXxzUhOOtjoS+6tcKOPu8bgCkouPNoXMU6xMKdw7kKsDty8QZMOXrWofjQx5PzB70Z0p0GhpiSwNAUZzp8Qwn01W9KVDa5KcU9BWEp1Lk5qudX1mfC5bru2SfsIacQ1xp12pwhe0iPadgTtR0i7Hpxo4LQ+Gqve7Ar1KDm1iuXH3gznz+reTPjqruSW3Vf2BUSr7TxKeG/3Mo5j/BG8rkq+J3aqROiRnVUntEbEer/+zsghZi8gr/O77iuUjubQnyxv0Ky4l/RH8nHpmAh0NPaZZUKGDjkLq5BxuMQrstpa7FhX+W/vzsXRcu+S0PXegE4HzIc80VPTJj+F35EtH5qRThqJH/cJcaJSp4L0JqZ5zCw16yS+sg8qqFeApZvcFBiYISh+DzI2nMwJgwY+TjTwdPWPhWsfC3HO2THBI446IVF3HTMREkFU+ERSz1eDFcPKHYyop0Af3FLogci/WAnVwTLSQoNgAJJj6tqsi1LerUmevvwFCP7aEfr+/nKMv9qHLOzj7CH9IErMxwcp3lVbQ8b8PAs1d+oHBMEL0W3NJipiA/WsHNSWGcpK14NUffxwx4dQZGiijD0YIKvFc6eQsrwdOi50auuc+AtrYYsm5PeQ0FbocYqzmjGqqS+zeB3XXs3COUlblHWRb/xhG6JKPSy8SMbTdrHcSOGYIHz0rwIaVaxFQ9Xf+GmA+NqQvh2KWZNRB9zrMkpnJC/zrRRDdEVr1qcpDvLgUeNgRa0GAsKZ7zXUVWcFoZoBwnGTPFgy0Vm3F51JjMHbJhiNbA1ooz5toPqqDenxRCTcSq3sg02aJ9kEOvq2tEJFZ1WOl6BuC6yrsUzbcVf0UWsYt97Qo2DeFQR0Mbnih5IlRPt8YFFBNK+E+orLB8EX/hqp2kqQmvif/75jQl73uwTiZzXpwewWgDYFmVsDgoPsOhNYViLPkXCumMW7QD74dwYZYPD8sn5FPHMoaqPqGWtEKt6Obf9selsJtfuuDkxn8Sr5lCzgJUhnjawQQa7zojtoNVsY7i+V4SMn+Aj4n/WvuXdn83+US8V7fO+R9LJK83vmuCIt7NG+vYeXovPXV+V1y/j/fQ4KxT15WOVVxt3pK5pbNnKz9KF/isPyFPoYuSkBi86iRe+NYNuiMoM1KAI3iiq6VDIGEq9MNvWyv5plCynMQoLMnLfojI39n2O902S+R6yUDud/uE7dkFixAWPe+XLtmwNw2Ekk63haqSB298CYMdZmIRiVmbnrvwenhE2BgKMyRBBM9sGAM=", + │ "page_age": "1 week ago", + │ "title": "ai - npm", + │ "type": "web_search_result", + │ "url": "https://www.npmjs.com/package/ai" + │ }, + │ { + │ "encrypted_content": "EogYCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDG2umNhfxWLLoy8/dxoMdD5PnOG5iue1z1pyIjBgsRxtJIke621D0LSSb5XKwrJyiEwaMc4S22/mcPM0QjCQ5snh6GATymqtv4py0M8qixeEEiNMxJ7UfbFWmN1i2n3scwxX0AMxku6FBUurr3Dfzh/W5nv/Vjok1BA3GPgc3i6Yp0xYDuQaMFJ5WOApQ+JnD5dBAv84qNPbKZD0BuW+62nPF8yD3mRs9fYLnA1J0NmMxCCw8aUF2J5vJjnmlIYb3vtsm3V4J2affLmgUcD1Yp/IGL+rGfbG0R1I5NWgJtsWglqYVHFl1VxVtYIDHbyWEed/R10MTKXCttY6Ew+mtpLbm9RDDzhkEsJ7Dl981G4uIFhfMe7FwQi9DbwQqB/TEKxNDbGE+Ba0UVEBIclhOCddVQxDwhkDbB7YPMTlZgeZgQXOSuV4sfj0OkLIR1tAb1+a+xQEB0a2e6npszXAwfMKRhI4zgBqhSbxwQbug1Qq0dt12S19oYMCkxOpsqpKn9fpB9durQhxtFEQ5WvI+RlH95ueDX2oLRv19kZ7uFc13poJhBZyRMiqu4LA9dDWZXyJi+sZYSSyUOhOuRtz4l6FKyW9Qp9P1yTVIqf803DC/7nhx5zHvPpRE0Fy4RQLO9BcY9Yr8LqK1lJeC8Ra6Wt2mqpY0bbufVj1upWJDIBbsUWzoPuq1Ar9LNuotdZq9HIZicSONp+VT/xJb35TE+9UKpr5KzcNh0BoWJcTb06v/TkSwp8QrS5iPQxKumNl61PcVVmtqn7bMYhTiBUT0byA0u9PtMaol5999UeQsqXzEin689lIPG/uvP7u2jUrUDuQRT1P1x4qm07DcaHRF5jOYQwemUmHdpsmvapyLO007uP9nkk5VomJFw3PRYfRLM7wLGKRlRmwVWCWLN7xOId1USTiRW6yYO9y2fqHfSUYLdKWjwK2eC8Pk+uwAX5rzf0yIKciVXZgbTb44Xzmy5286Yz4E/b2Lbp1BHrMHzS5Xy1ntBPCpArhq7SD+WbWjYaA8ycGm4yWSeKOtHNZV9cUSJoZR8+yHowcF8fsUT1bFp0aYnCh4YKhr8DiEHbOaOp2f92FXSEY1ZLLFj5TwO9q6bBXaMUQk68zSuBRhaePL17qX3A6ssxzPfDJ+xUzO1D8L/j7R9PQJkfp7qnQYWc/XPEn+KCjugNxFAk/AVRGp+FKGEHGJnKncxYy/UfeI+tb6RNITzJPiRYhZOPp1oUJMF1PUOWbdKQkePfhE8Z9C/wXGZFVa85V6FrrAD6zCIB6E4fj8Zjk2V0RqMJgFLR0BDHeEgzZ5909H90eNsXcx1KSXVogD5oYpM2UDnJkVAGzrk350q80Wntt+RCmGsr5g9g8MDCfuf32fMajx3e3S+qwzF9BqOm514eTV7XQnv6iqG0s33bG3hZi9lUCi0vNAHnoWaYCz3aKJl59hxIn6iDARtNRuH2UAQ7WeT/9CnkZoRPjDUFE1r2nP5cYApYYlD1x4JN08Bqe7pYzOkrFYhqSOLYkBMeKDmB3zXRRZcbsWo1oK6Wlw1Hx/VnVU6V2Ecq2GTh3TOi0jChLLR+pzWwiOG3DWTTBiR85hOmK4ljq7S5j79ElZ6Ae+BFSE0FJjJwsIGYOrg9X0YOyFjqQ3dyVWtzm3W2x4erWvoCwDDFU0J95EGLUCN9EAga7uGnD0rf6OCLyV1F26GmL7PB/zRjLweaWG7L+96aFnb3YtE8BQjxbuQHQZIoitfVN2qWeYbDNBtPpMlB+IaYARipqqWUYXCvY+1QkYke1vhBeICN+Cz/9ZDaynpOgqIddRZUdzgoUIkDJN200mQ7cbeZr13fj4XiEgQDw4remIICOYobO7TgqfBBRFiyeZZxRFtuqAdzOMQbFTk4lLBDmf1RMPFmZbOUDtRK/Wlbl8zaXEwyfYfzNY5rnMuQaII6U3p/6GCSgGQ2dzlc2rCrnjRDN7hbZhC5ggNCGNptJ37ZLdx0EZjiPUT4DbYNNW7pToWitok/P0Z7Ldi4sSk9szjoEIo8T9IkCMXOmeQ+9CnrqXaWRRJQ6ZSPUYOo0Y2w4w48+D+e9VEgdswsqVx6L/I7gyZz7CBTkfsg6UkcVdKSe6JFWbrMKQhQVkgpJO86A82TVMsWOMkm2XqW3XWaWZHU3HJ429lQHD6HThloOHV27bNOdxJFm1AHgGUeY/DwnqoncrVS7ZnGfpwfLw/LttxMQXp8m0dAsVFDXF7Phx4mQinsA3W5uDvWalZc4WQmGw0osA2RKQjvAuxJ99MO7ITTZeqLeLZ3wN0Kj3avcmMHgZ6WfFVpZ/FuZp5126d+Hwq9c7T+1vw/0PsxAgBOS/rFT5Me9Uw0gwhpL3CSEvhYOMnZYvs/V6iV7+jxMDWjDsnxkjkxC5FEl+HZNeMfPO9p+dl2JD7NBF1LtKn9cpmjeFBSqBCQk7rDV6MJIUN11rH2rxpPzelq/7KByeGdX/5d1aslY8xIuBqpc3TV7jhejEZN8hznUxOH1qoFP0jc48DD1mDOuySAdBCKdIWN0gjKCxr8z5CU6CmuEXwGSzaJ8S9eHOEoOKwU6mZvwj0VdhMHJxSIvT6QkjM2U/1LDirtBY7Ij7kT+xvogWSRjypIg+xTXsl5C0t5/KO07Edn3doD3KjkoA8txdQy5/2EUEuFXZd2LK7D4uX6+oA0iXmHM0xMBYIbWzMtPmXrJ3jaitbvgL9ChIkbtkVygSm38sJCUbXZhaYqnFxLZonLLP7TAvgoXxEgfTJ67n4MvhT5NTaYijKItGPU3vVBu1MkqbDMeXTPED/JWfeg02AguLg/ZwoPuqsKtuIaN1saR74UawIiX0rCDFfLpzzeTOJvw9DfgfowDHNs9P/Z4w3wlrIobDJpHfHTg5naSUTUyNYSmF7GnMVATwFoBtYem4JtKnnvgqUeQ1zkMUIGHv6/eKNkN249gQ10W2wbEFA0P1SNjm+E7IBZDo0+8Yg/NQuHpjdjmRb186sR/O7/Z+Cm0skbcTbm//8t3Zw2Fp5LY1HvHOVvnfvtcgCwJvCnkY0orZG7V1KcuBa0yl8fWoSjkluoITpFw1tm34msoJt6YZyJ2Wb9GL2Q+ElaydG1TGbI9541bjYjkGAbzvDrqIJjBQXkjvNskjY0XrgxFkVdy8yCbeqP/PUN+t5IuFnmqByWLXH03Zp8apNShNl9xZLGWw/PlEZF547S6vuQ7Z+pK6H4kBAb+4BKjXYdtwt3i1lfJbTLJ1ws2pxJ8WWrwLwEVH8dAVAQ/1NyhKeVHiExqZtaOwfB4IpCmjoTElUSeAAGhjUvShQTV8yzGajPgqSpMMraBuuHOSQyzHMCMlj2cqQyLiRMBsMgD2hJ2qGIYeXcsibJSBrtVZYKQ798e3mfGTCR6/ztEimurdi+ctNbUcVAri8437x/FA6yqZBBwPDyoYsdi+N/bcN+COFaryk5nWn6qd9J6NOXMfO5xmGWExbVLl5c++2iFcIQgHOyKTAXYFPB0X742uLeZkrdpwCxYr4bkYiwEzQ84/wn4QSXj+p6oijtb86g5FsnOeLb10BNe7SCD/xdT1OijIJmZK7diOd7ESs7GYo3ElGTwcyTEkxNpLE3MqtUKcogn2bJxite+L3DkUDturTzaPvVJb7+njiNp2dgkOmxx+ElnUD57iO3y+OqYcdg619kX2PG2DaBEK5nNFlaB2vqTLmAvRZHrS8An9/dy2eB+dhgwThgHoxyq0KyoZt/8m5IPmg7Zikdfy4kADOUHenR8m5MLwdlhkJAAjfZt23uXMw1L+XG2iKSx4jPsG3U6kDrF3S24lnAn3oflChUcTEeztocCc2JkREwa4HQqM10EdbFlprDjAOfHdsRTFpUIx+tASJJY3nlZToWdGkNVDHn02VclXdpuJAtNHoD5AFeL7B4iMpbSZ7Ddq/UuHq94uXc+pidZsxytF19sQzFxqkSlOXRelZhljS6pPU/bCcZizuk5k3fvJJtgY+EI8LTOiRZbWMT3vb7ATfu3Uo23+xavr+9hAmM9a2iC1xtY/PGFVjF8ttQYAw==", + │ "page_age": "1 week ago", + │ "title": "Google Cloud release notes | Google Cloud Documentation", + │ "type": "web_search_result", + │ "url": "https://docs.cloud.google.com/release-notes" + │ } + │ ], + │ "tool_use_id": "srvtoolu_01YQZbh8cGvpK2NTYcEos9aY", + │ "type": "web_search_tool_result" + │ }, + │ { + │ "citations": [ + │ { + │ "cited_text": "First seen by Releasebot: May 13, 2026 · Vercel AI SDK by Vercel · Vercel AI SDK updates dependencies in a patch release, including [email protected]....", + │ "encrypted_index": "Eo8BCioIDxgCIiQyNzc5NjY2OC03MzUxLTQwYWMtYWNjNC0wMjRhZWU4OTk1YTUSDCeDAdQWuM5Xl9vRsRoMAY1RgpqXY4nSMSnPIjBoPuTTCtOSZ4HsXYn0dsuOY8UkBVMQboGZ/AKhc/KnlJ769yI3mUTrJuurulBia3AqE767bZVGoPU4BPvfsD2wrDbjLxYYBA==", + │ "title": "Vercel AI SDK Updates by Vercel - May 2026 - Releasebot", + │ "type": "web_search_result_location", + │ "url": "https://releasebot.io/updates/vercel/vercel-ai" + │ } + │ ], + │ "text": "Vercel AI SDK released a patch update on May 13, 2026.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 256, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0, + │ "tool_choice": { + │ "name": "web_search", + │ "type": "tool" + │ }, + │ "tools": [ + │ { + │ "max_uses": 1, + │ "name": "web_search", + │ "type": "web_search_20250305" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 75, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 16216, + │ "server_tool_use_web_fetch_requests": 0, + │ "server_tool_use_web_search_requests": 1, + │ "time_to_first_token": 0, + │ "tokens": 16291 + │ } + ├── anthropic-stream-thinking-operation + │ metadata: { + │ "operation": "stream-thinking", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with the number only.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "signature": "", + │ "thinking": "The user is asking for the sum of 2+2, which is 4. They want only the number as a reply.", + │ "type": "thinking" + │ }, + │ { + │ "text": "4", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 2048, + │ "model": "claude-sonnet-4-5-20250929", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 1, + │ "thinking": { + │ "budget_tokens": 1024, + │ "type": "enabled" + │ } + │ } + │ metrics: { + │ "completion_tokens": 39, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 49, + │ "time_to_first_token": 0, + │ "tokens": 88 + │ } + ├── anthropic-beta-create-operation + │ metadata: { + │ "operation": "beta-create", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly BETA.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "BETA", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 24, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 18 + │ } + ├── anthropic-beta-stream-operation + │ metadata: { + │ "operation": "beta-stream", + │ "testRunId": "" + │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "role": "user" + │ } + │ ] + │ output: "1 one\n2 two\n3 three" + │ metadata: { + │ "max_tokens": 32, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_details": null, + │ "stop_reason": "end_turn", + │ "stop_sequence": null, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 15, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 24, + │ "time_to_first_token": 0, + │ "tokens": 39 + │ } + └── anthropic-beta-tool-runner-operation + metadata: { + "operation": "beta-tool-runner", + "testRunId": "" + } + └── anthropic.beta.messages.toolRunner [task] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "anthropic_tool_runner_iterations": 2, + "max_iterations": 3, + "max_tokens": 128, + "model": "claude-haiku-4-5", + "operation": "toolRunner", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "temperature": 0, + "tool_names": [ + "get_weather" + ] + } + metrics: { + "completion_tokens": 72, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1294, + "time_to_first_token": 0, + "tokens": 1366 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "location": "Paris, France" + │ }, + │ "name": "get_weather", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "max_tokens": 128, + │ "model": "claude-haiku-4-5", + │ "provider": "anthropic", + │ "stop_reason": "tool_use", + │ "stop_sequence": null, + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "description": "Get the current weather in a given location", + │ "input_schema": { + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ }, + │ "name": "get_weather" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 56, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 607, + │ "time_to_first_token": 0, + │ "tokens": 663 + │ } + ├── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: "The weather in Paris, France is 18C and sunny." + │ metadata: { + │ "gen_ai.tool.name": "get_weather", + │ "provider": "anthropic" + │ } + │ └── get_weather.lookup + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the get_weather tool exactly once for Paris, France. After you receive the tool result, reply with exactly that tool result text and do not call any tools again.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "The weather in Paris, France is 18C and sunny.", + "tool_use_id": "toolu_01REdkoKf9Cn5nKu44AT439h", + "type": "tool_result" + } + ], + "role": "user" + } + ] + output: { + "content": [ + { + "text": "The weather in Paris, France is 18C and sunny.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "max_tokens": 128, + "model": "claude-haiku-4-5", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null, + "stream": false, + "temperature": 0, + "tools": [ + { + "description": "Get the current weather in a given location", + "input_schema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + }, + "name": "get_weather" + } + ] + } + metrics: { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 687, + "time_to_first_token": 0, + "tokens": 703 + } diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts index ba8993dc4..cf6aa4eb0 100644 --- a/e2e/scenarios/anthropic-instrumentation/assertions.ts +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -1,17 +1,17 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -280,12 +280,12 @@ function summarizeAnthropicPayload(event: CapturedLogEvent): Json { return summary; } -function buildSpanSummary( +function snapshotEvents( events: CapturedLogEvent[], supportsBetaMessages: boolean, supportsBetaToolRunner: boolean, supportsThinking: boolean, -): Json { +): CapturedLogEvent[] { const createOperation = findLatestSpan(events, "anthropic-create-operation"); const attachmentOperation = findLatestSpan( events, @@ -338,194 +338,90 @@ function buildSpanSummary( ["get_weather.lookup"], ); - return normalizeForSnapshot( - [ - findLatestSpan(events, ROOT_NAME), - createOperation, - findAnthropicSpan(events, createOperation?.span.id, [ - "anthropic.messages.create", - ]), - attachmentOperation, - findAnthropicSpan(events, attachmentOperation?.span.id, [ - "anthropic.messages.create", - ]), - streamOperation, - findAnthropicSpan(events, streamOperation?.span.id, [ - "anthropic.messages.create", - ]), - withResponseOperation, - findAnthropicSpan(events, withResponseOperation?.span.id, [ - "anthropic.messages.create", - ]), - toolStreamOperation, - findAnthropicSpan(events, toolStreamOperation?.span.id, [ - "anthropic.messages.create", - ]), - toolOperation, - findAnthropicSpan(events, toolOperation?.span.id, [ - "anthropic.messages.create", - ]), - ...(supportsThinking - ? [ - thinkingStreamOperation, - findAnthropicSpan(events, thinkingStreamOperation?.span.id, [ - "anthropic.messages.create", - ]), - ] - : []), - ...(supportsBetaMessages - ? [ - betaCreateOperation, - findAnthropicSpan(events, betaCreateOperation?.span.id, [ - "anthropic.messages.create", - "anthropic.beta.messages.create", - ]), - betaStreamOperation, - findAnthropicSpan(events, betaStreamOperation?.span.id, [ - "anthropic.messages.create", - "anthropic.beta.messages.create", - ]), - ...(supportsBetaToolRunner - ? [ - betaToolRunnerOperation, - betaToolRunnerSpan, - ...betaToolRunnerToolSpans, - ...betaToolRunnerToolChildSpans, - ...betaToolRunnerChildSpans, - ] - : []), - ] - : []), - ].map((event) => - summarizeWrapperContract(event!, [ - "provider", - "model", - "operation", - "scenario", - "anthropic_tool_runner_iterations", - ]), - ) as Json, - ); + return [ + findLatestSpan(events, ROOT_NAME), + createOperation, + findAnthropicSpan(events, createOperation?.span.id, [ + "anthropic.messages.create", + ]), + attachmentOperation, + findAnthropicSpan(events, attachmentOperation?.span.id, [ + "anthropic.messages.create", + ]), + streamOperation, + findAnthropicSpan(events, streamOperation?.span.id, [ + "anthropic.messages.create", + ]), + withResponseOperation, + findAnthropicSpan(events, withResponseOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolStreamOperation, + findAnthropicSpan(events, toolStreamOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolOperation, + findAnthropicSpan(events, toolOperation?.span.id, [ + "anthropic.messages.create", + ]), + ...(supportsThinking + ? [ + thinkingStreamOperation, + findAnthropicSpan(events, thinkingStreamOperation?.span.id, [ + "anthropic.messages.create", + ]), + ] + : []), + ...(supportsBetaMessages + ? [ + betaCreateOperation, + findAnthropicSpan(events, betaCreateOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + betaStreamOperation, + findAnthropicSpan(events, betaStreamOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + ...(supportsBetaToolRunner + ? [ + betaToolRunnerOperation, + betaToolRunnerSpan, + ...betaToolRunnerToolSpans, + ...betaToolRunnerToolChildSpans, + ...betaToolRunnerChildSpans, + ] + : []), + ] + : []), + ].map((event) => event!); } -function buildPayloadSummary( +function buildSpanTree( events: CapturedLogEvent[], supportsBetaMessages: boolean, supportsBetaToolRunner: boolean, supportsThinking: boolean, -): Json { - const createOperation = findLatestSpan(events, "anthropic-create-operation"); - const attachmentOperation = findLatestSpan( - events, - "anthropic-attachment-operation", - ); - const streamOperation = findLatestSpan(events, "anthropic-stream-operation"); - const withResponseOperation = findLatestSpan( - events, - "anthropic-stream-with-response-operation", - ); - const toolStreamOperation = findLatestSpan( - events, - "anthropic-stream-tool-operation", - ); - const toolOperation = findLatestSpan(events, "anthropic-tool-operation"); - const thinkingStreamOperation = findLatestSpan( - events, - "anthropic-stream-thinking-operation", - ); - const betaCreateOperation = findLatestSpan( - events, - "anthropic-beta-create-operation", - ); - const betaStreamOperation = findLatestSpan( - events, - "anthropic-beta-stream-operation", - ); - const betaToolRunnerOperation = findLatestSpan( - events, - "anthropic-beta-tool-runner-operation", - ); - const betaToolRunnerSpan = findAnthropicSpan( - events, - betaToolRunnerOperation?.span.id, - ["anthropic.beta.messages.toolRunner"], - ); - const betaToolRunnerChildSpans = findAnthropicSpans( - events, - betaToolRunnerSpan?.span.id, - ["anthropic.messages.create", "anthropic.beta.messages.create"], - ); - const betaToolRunnerToolSpans = findAnthropicSpans( +): SpanTreeEntry[] { + return snapshotEvents( events, - betaToolRunnerSpan?.span.id, - ["tool: get_weather"], - ); - const betaToolRunnerToolChildSpans = findAnthropicSpans( - events, - betaToolRunnerToolSpans[0]?.span.id, - ["get_weather.lookup"], - ); - - return normalizeForSnapshot( - [ - findLatestSpan(events, ROOT_NAME), - createOperation, - findAnthropicSpan(events, createOperation?.span.id, [ - "anthropic.messages.create", - ]), - attachmentOperation, - findAnthropicSpan(events, attachmentOperation?.span.id, [ - "anthropic.messages.create", - ]), - streamOperation, - findAnthropicSpan(events, streamOperation?.span.id, [ - "anthropic.messages.create", - ]), - withResponseOperation, - findAnthropicSpan(events, withResponseOperation?.span.id, [ - "anthropic.messages.create", - ]), - toolStreamOperation, - findAnthropicSpan(events, toolStreamOperation?.span.id, [ - "anthropic.messages.create", - ]), - toolOperation, - findAnthropicSpan(events, toolOperation?.span.id, [ - "anthropic.messages.create", - ]), - ...(supportsThinking - ? [ - thinkingStreamOperation, - findAnthropicSpan(events, thinkingStreamOperation?.span.id, [ - "anthropic.messages.create", - ]), - ] - : []), - ...(supportsBetaMessages - ? [ - betaCreateOperation, - findAnthropicSpan(events, betaCreateOperation?.span.id, [ - "anthropic.messages.create", - "anthropic.beta.messages.create", - ]), - betaStreamOperation, - findAnthropicSpan(events, betaStreamOperation?.span.id, [ - "anthropic.messages.create", - "anthropic.beta.messages.create", - ]), - ...(supportsBetaToolRunner - ? [ - betaToolRunnerOperation, - betaToolRunnerSpan, - ...betaToolRunnerToolSpans, - ...betaToolRunnerToolChildSpans, - ...betaToolRunnerChildSpans, - ] - : []), - ] - : []), - ].map((event) => summarizeAnthropicPayload(event!)) as Json, - ); + supportsBetaMessages, + supportsBetaToolRunner, + supportsThinking, + ).map((event) => { + const summary = summarizeAnthropicPayload(event) as Record; + const { name: _name, type: _type, ...fields } = summary; + + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + }, + name: typeof summary.name === "string" ? summary.name : event.span.name, + }; + }); } export function defineAnthropicInstrumentationAssertions(options: { @@ -541,11 +437,7 @@ export function defineAnthropicInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -930,32 +822,8 @@ export function defineAnthropicInstrumentationAssertions(options: { } } - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildSpanSummary( - events, - options.supportsBetaMessages, - options.supportsBetaToolRunner, - options.supportsThinking, - ), - ), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildPayloadSummary( - events, - options.supportsBetaMessages, - options.supportsBetaToolRunner, - options.supportsThinking, - ), - ), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.json new file mode 100644 index 000000000..b2f458745 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01VxxkUqbM8wfd3hj1SmgRLd", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq8ECmMIDRgCKkDadtLAudj92ZxoHFr02fRRfLdtYHObOcDiQF6ul57eNMrz7rjJRJ3QrzBGk1NXMBH0GFaqLSi+RlFFxxOIZdVHMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN3AKEriP43sm8rrBRoMQ0btkN0jvb2epKkNIjC7ldUV03HnOSw24KLaHqqU9qwXp/dctQorajSK1JEtAsSaJbi/Vb1CLtVDnYNW9sUq+QJRBOiGixZwhLwkui9WqHYS3vhipPf45xZ/sCcQrEvSC1PTuLt6kOpXnyOxqSAKk/kGlvdzNyrkKJxzUG8y7MlCLVTs3u+JE8pnjUc1PRksWAGYjWpjnTaaVfiozn+OmVC0s9U3SKpk6RMlayil9ZfitdcU1/H6SIE6FMVvCo8YRA5y/G12NMhpkUlqxcnaKZ18dDJNyjoaEVuMF0NZdt93rDhYu6hFl/8RyMbrKTl7GQlniytPbVDlQiirQClhc904iff8vNS6XH7q9NoebDZS76d+OcftyI4PBVjZqZNn203YOMatyfzTgroY34m9HjYVrFoN1vovKeH33T91p3PwxI8mQ65PGHKWd3vPUZ84Hl/Yh9zTtcyIvwBhqOPWoC2/4OA7B65vLt5qPAQKHvWCZQL0m21Xb9A2O0KcW89GJzHm1XpF1YXpWU4DO2K6/yuEYBwU6r3F8P3dTZrv6yfItQWPFOMjD5Bap780cDeDJ12FaLG6mMJdCBgB", + "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory,\" so I need to use the mcp__calculator__calculator tool to perform this calculation.\n\nLooking at the calculator tool parameters:\n- operation: \"multiply\" (required)\n- a: 15 (required)\n- b: 7 (required)\n\nI'll call the calculator with these parameters.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 1, + "prompt_cached_tokens": 18783, + "prompt_tokens": 18793, + "tokens": 18794 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoICCmMIDRgCKkBuOEGYvbAI/nasJ5uoeL24yzLiIeRdwQTt4HJBellK8Aa4yBGKOZWZspHlpdQtB/+d1zkLOcwJby3W7zyAuW/DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCkjTiBiVcs0ZAyYJxoM2JHhZM0wwqkYqmDJIjDR2hf6Tcli+V5NKQaKTlXgE2bdYv/e7PnaemvhIqr8AZ03MCLepewbbvPf6coBZqYqTah0DGLlwMgTGOox+vt8Vp3zXO4bt6BXVTQsWLCAYMVYXMGEEG4u6QhnAt48KBoIMeN3Jh7TpWh/tbmmN14JjUFbkq3cf6+8hWiDXhkEGAE=", + "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The result of multiplying 15 by 7 is **105**.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 237, + "prompt_cache_creation_tokens": 223, + "prompt_cached_tokens": 37566, + "prompt_tokens": 37797, + "tokens": 38034 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "The result of multiplying 15 by 7 is **105**.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Et8CCmMIDRgCKkC4KFlIDcPjqXKV8EX7ZYp0Oq5ZG2Fl4ESHlmncD4Np+A7nnLohlO8SyZdbcTLy6hhx5h3ZlJp4Q0MXFATUSyTLMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPeXK8/xbZe/mkPmjhoMf/e6FqGNR5+7fYxMIjC8uQkLsE0n0oUNbgnntg//2PV0shjx6lPJ6Kuj5cmUGh06uBaxmHm7IKFgVgFeGZMqqQF6rsHMOKjv7zs0qV1CPLS8rA++ZXCD4QtB58rNrFN1MIscrsGTVRdHanQzC5mhoaKhuMPJQ+YwO5qBrhcTYJw/LE0tsZrkLLRFVu+bn4zbaGQ/mFd3v7joOSXjcNlGse6dORxO1JFWcGIHrqxsi39mwdrJtsS/n1/IlrhABWCgovtZf0nBWAIKSj/CWfZFC7SBG0Tblv7fMeDAsnhEQbiAo8wusLXnT1SxGAE=", + "thinking": "The user has sent me just \"Part 1\" without any specific task or question. This seems incomplete. I should ask them what they'd like me to help them with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 161, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18660, + "tokens": 18821 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq8DCmMIDRgCKkAsX1krZGggNLkav1y8f1xlySOEtQFt7PJaDzTNcDHCLiNbJHmg1H9X8j8b4dNEema3PvtJvp9QfQXFFX5MvsV/MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHdvoxYPsTAAfYpbgxoMNt2Ae+wHogoCxKxrIjCKTZQ3bl//cEeNG261osG00Jb8/m33Bf+IyKJiTeWn7Am2JtmYlb/pnmki76ayAo0q+QFRk3lTz65m+rnRwVhErO3IbQz7wMC+yV7FXUcrjlijlkcnGTElHar43PwAsTtQJIm1W80eUnoMA4oKslBYknpD5Cwc9dm//MYw5Qtva3jbAj+q+NTDVLla347ok5PBG8XKqaMUkjao/8u7cNkWOfpstBzF8vqOaveFv4msP+tlu1mb3zF1v/2UKXc8pNdojJ/NLbwjNtTHIq/tjvzlaqu77UGkhdcUpmbZK5612upPREcSc79EUcVZFkoxHdt3BPAtccCKENwSlx/OPZ+sedE+0KELr9t/e5jqOZYVxP7pYz7xMAxv2n1D9zO+JwSq4MR0FfAjW4e5PG8YAQ==", + "thinking": "The user has now sent \"Part 2\", which still doesn't give me any actual task or question. They seem to be sending numbered messages without context. I should ask them to provide the complete information about what they need help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 174, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18834, + "tokens": 18850 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "ErAECmMIDRgCKkB+s9pCYuRUueJD29vz1evjL5O/fb/4HPxVeQ4GE9uV4spyVVS2MnyioFyJr/nMHesx702uruFoVYv4+zNVFrGtMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOdEIAziK8tIRorTtxoMRci5Re6Cj1+d5RE4IjAPnkmSWVsmJxYeBKm/BEjyJq0IakmSbUJ3U2IrGO5FWv/4z1iTdCweDLgGuZnvHjUq+gLAuEQltv8fB8YqpTPMBIAS237ScFwpTCfhnz/ie7VjEOk2G68I6v3FKi2RYkjtsP4sUDkOAGsaDnek3GP7edOvodB+jdkYhJeSvGGke0Lbli9ZJdWORgNf7aZCZD41enJw3cIgjFOwxlWJHxtmFX+ZEmTkzTXFjlGJWbJwwLwpCiHVMFQEexeuDF5VcEz1TdpTSrdr8rH0OM4Uf/VdMRE4a5SovaJA6TFnLEceyTXpIoG7ct6LyMst2Nt2JvCLPBaYLN46HbuKXZvjF5QU0PSJAeeAJIm2NMmVytew87gR6dR9fjmUhbBKVk4ZStmfILvKqeL9vpDlVKknXFqp5crkbeN9W6aBpZGVhIWKePUs3V7rezhLBrtGzxF+ewxBtoo5xAHVq69llFz+q4R9A4SCBGrJbtHS0SAxM003TCtvt5TaS0rK2zotf4YMAZalWhZm71W1ERVotLdtqhA+4ii6gvKihKHG+hW23xqWQzqvEYEVJzU+jP0HAPcYAQ==", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. They explicitly say not to solve it myself, so I need to use the Agent tool with subagent_type \"math-expert\" and have that agent use the calculator tool to add 15 and 27.\n\nLet me create a task that asks the math-expert agent to add 15 and 27 using the calculator tool.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18811, + "prompt_tokens": 18821, + "tokens": 18824 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 using calculator", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01BUZHbp64XJFUZRU2vuUrJ9", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cache_creation_tokens": 9148, + "prompt_cached_tokens": 12880, + "prompt_tokens": 22031, + "tokens": 22034 + } + } + ], + "input": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "output": [ + { + "text": "The result of adding 15 and 27 is 42.", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 using calculator", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + "claude_agent_sdk.total_tokens": 22037 + } + } + ], + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The result of adding 15 and 27 is 42.", + "type": "text" + } + ], + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22179, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 127 + }, + "cache_creation_input_tokens": 127, + "cache_read_input_tokens": 22028, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 18, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtwCCmMIDRgCKkCbSWnZ5mk49Ij0H4p1KEDanAwnJP8zG2I8b2HMlpnmrnccGoiO6zt3Kq911LYtcgCeBn5NOIjt+RFZFDKU0UoxMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLuHT/DzOWhhV+Br3RoMoj29xhlCaRtfJM19IjCC7sceqFG4VYeOEiM3vY9FZ+PFe+y+SQbUzf53DefX4SimPWyUgQSQj3X4ovy/NZUqpgGEYmlevBUvBscb6jCJiLOBrz6sHEPCIaS0JlvlEadMNO3E5Y/crefLeaxfF1GRad4rbGWIcbbuhsbk4rQmpA5m1nw2zd2raSfwYqchoPnY2Q2ZNjPAizHngVmMyvt1UowJxsC0LTzGHUpk+uHRdmGMVtTJgJRizi7NhvwmM4q/PYzDttnJSJ69rU5fVb8/CIpgRC3M4+kVcldwC0vA+7vkJxSUgRyBGAE=", + "thinking": "Great! The math-expert agent successfully used the calculator tool to add 15 and 27. The result is 42, which is correct. I'll report this to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 315, + "prompt_cache_creation_tokens": 318, + "prompt_cached_tokens": 37622, + "prompt_tokens": 37948, + "tokens": 38263 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eu0DCmQIDRgCKkCOpbxDq3fHTCBNR/vFiVrgZg35kTKAwFMsKMpfSRipC241ois8Tank+XxPuWaOSap9mIzdpETFLgexwY4UIStnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgxfoBC8hOxqJzMxeZ8aDOuq83KbtOpng4JqzSIw5+c/dGSrxJsP+a/KIj1hkErEnabsaq72PkvK3qwgYXi2lsDxs+hX4ANhJvFbg94tKrYCCTBXTZqqvUvNnWrEgTPwHhBe09AutUv7sMMOHuc9MXAj/Q7OY7xd0Y5Dni+8R0vdBUV7ykR9S6sYrbidoGsrOIN4/sBX3Z0FsrxT15oZj6K6D3d1yaesCDwFGyNpboLIIkqEav/H7qasuN1RRx4mUrEOJsVMuXY6upMWGzJKSzFTQ/rDnXktl5gmbHy0H0sHwVVUw/1J9S2jRh4g7XAvFEOZp/Zt322xya27TC1cAO31PemCickN/n1F4C+Cvd2UFVNn1PvgvzU0g3fK4WouO9yCf3Vocp/uYrBf0lq9LGrVGhW/vGYUe57n8joHzWyQV4vyVyLsnXJtmy71QENuNP2lCzct3n4mXddRRqJQxdpKRhqA29SzrlWrp/Y5SOWBmbZ0W24Hd5gfi9Zw+2eGgYlJmMFuuRgB", + "thinking": "The user is instructing me to call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". They're being very explicit that I should not call Bash myself and should not answer with text myself - I must delegate to the echo sub-agent.\n\nLet me follow these instructions exactly.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 18711, + "prompt_tokens": 18721, + "tokens": 18725 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 49, + "prompt_cache_creation_tokens": 12519, + "prompt_tokens": 12522, + "tokens": 12571 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01UePBWB4V5dqHqpS2KiqqXH", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Please echo a greeting message", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Echo hello message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + "claude_agent_sdk.total_tokens": 12571 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Please echo a greeting message", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12612, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 83 + }, + "cache_creation_input_tokens": 83, + "cache_read_input_tokens": 12519, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpwGCmQIDRgCKkCksSUYYvmFw41lDV38zAUKG+83X7TBU/YzYmHYMyq5Aq6UbGazCBQbDfQrnvVcmKlQteKdmGESruKJlpZq92vnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgyjH44y9M2K401hdesaDFhA68JEWyW1UKlUQiIwhRlvfLb30g9/+XtIC7a396RgfQAi6xmU6EKoKoUGvG8cdmhcc4Asn+TwW2wr3KXiKuUEmrgQAkBCoDpv9XKMBx+BxuvQ5eqJM1AVBrduqFZzBLhBbs9SAYdSAjXpiI1DP9dVV33FbtLZETurq3rQX90esLk+r+sTEiDxPtzl2EmTUjnHfCYjNbjM/CJRY5mr4hIuinrhqwkG01enWcF/HqReLb6ay4yiqoItBFThpMhthCshRoi1bcp558knW10lHkYvhKG5SsrKbvcKTzklbgKAaqvACyDigdvJ0Me3bt5Q0YI0rKHLWMXZXmLpDOUG/RY8IWg55HBZbsVtWlR1TVQ//o/s1N91Un6oJa5XLGQdzyDiO1iqkPhf5XSejZos69m8tbs0Ee+8mx2l3yOx+hWY3tk5g4MUk9z7Cd95Zt0TAfk8R+hMrX5GPzsoh9bGx+l6SllVptbGIypey4Yt66Cw6wkdNYg2oc4udyrCd27vjjW91MZ63JIxHp75hIfiBBDDrDvuSq7Z3r/K3EDhotwXkOMM0rN78DNsA2jbgeoZmgOOyAE+aRKMVRquHw7IfOMB4TSLFBaY+EaBMf+FjCgWn7SNGo/rJavhHjcU1NrVHxnLnqqxW7eO/A/iT96lGWQ1OpDHZc5FgggBJLQ6ylddjibH58hNXdzTA/2pK7pFgqz1mzKNjBMbJiAZ6jvPDdHZ3d/bSg+QxDeA9V/wLW/75JuVZpD6Tt6qZq9urHXJgqCkN+/kMBw1bui3cJqGSnMvP6LRAqVItnhTvClEEAgYRCyHt93BSIeUPHuIeEJSVk/xKzGGHgYHtlHhKEQylN5bciwwz2pTbzxPmaOQ34Rq8Po2qHkO1L2ZwmE/qaRjRgMpUYHmIRgB", + "thinking": "The echo agent has completed. According to the instructions, I was supposed to delegate to the echo sub-agent and not answer with text myself. The agent has completed its task. The system says the result is \"done\" with agent ID ab0746ef3684d1f80.\n\nSince the user instructed me not to answer with text myself and to delegate to the echo sub-agent, and that's what I did, I should probably just confirm that the task was completed or pass along the result. However, the function result doesn't show what the agent actually echoed. Let me just confirm the task was delegated and completed as requested.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed the greeting task.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 268, + "prompt_cache_creation_tokens": 246, + "prompt_cached_tokens": 37422, + "prompt_tokens": 37676, + "tokens": 37944 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed the greeting task.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01CMsRz4pP2m3xfVrPNDKi3r", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpoECmMIDRgCKkC9QeL6AxsMeyrD5cHHu1Tm5rpsHLRj52XKstohEXH8dyOHFIoE5T6D3NMcDG94eGws7KXEmWT30YNQWMP14ks0MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPC5dWnk783Es6mOxhoMEfYMZtnipe97ysV3IjDXPoM0YpVchmgiN64yfoZeaxtvxsJhWRiaSveKSrVLamuhLo1mKHrnl90F72S62c8q5AJWq/fwBQgUjbsuUmjXmhHUORkCgLjzztmhN4YjIiHrt8/p4bIl4f1Ml/C7Fe1gE8s27x1+PJlySQoCgCa9rbhJHJo22S/UAfwMpRFqCV7+kPRwv9JZdMXVGPsAQXihEaiKyt2hp4UDoHpYDDsWxskcKtxcfHc4KS4VPIPTZH6ARIrrrRcpgn/hc6gFf44xiU65+CAX13Iu5hF7yzL24ZOAzjiGiRLhizHMnic2tECaPRoBb6Yi8Noie9Tutv+A3Vlee9R9LbfMhBFunbYYF3S+eCKCrg+SapNkZG7WFD6c+ZnZhyYSJc3yudt1D4syr8kwDaOZ8JIVuC2JA+wKhyujuBvooS3gNNaOdBvgtc4c8sYlLvIGkfUmBAlCTUfcruPuG5lDh9NRRheIDgN5XJUSRNUkiM+RUCrguyMPR6GY+xv+bpszZrbBZxFkIH3T4YTrfi0MIgX7zXQl0X26GvVn6cCwZBgB", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly state \"Do not recover from the error.\" This is asking me to attempt a mathematically invalid operation (division by zero).\n\nI should follow the user's instruction and attempt this operation. Let me call the calculator tool with operation \"divide\", a=2, b=0.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18794, + "tokens": 18802 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpUDCmMIDRgCKkDR46Ny2JTznygtWJRCCliED4QbRa7fBafCWMqq3Y49sOAvTDXA6T8Lt4iK2G/gn7VDicKOj9yc99O5wtwoG0j7MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOzXlcCXhIuTuTpoZRoM8g8rJArlcfyzlG32IjBPIqicvTdNi18FzFWs8Jq51Tk/AtdOp09pFj8R46e35TtK8kzGGojN6qU3A4Nz8Ikq3wEnAiWddUgsGA9Ku/TeYxPhtL+6ZHW0Dg/S3E9jaKp0LsuoMl+5BvurRRy4FQj46d8h7IrL6pCK6icO8hBH7hPSMwF9vTV9+gHW/06y5vw/qSdsrcbnCiKlQj4nWEU/RFjI6XeFmdhJnSt83O73DsYioFuY49ilOmdzFl5InTqC8gpGsqc6Ml4Fm2sxa8WD+KQq3oVyRdSaI3iNZ02w2IBrkrK5SqvmbuQMF0iFG727fiksW84Mz+RjBBYEEL4FTEf/2P3McWJHVoKiooZdJWBtIu73m7yidEe44nW6E3suGAE=", + "thinking": "The calculator tool returned an error: \"division by zero\". The user explicitly asked me not to recover from the error, so I should just report this error without attempting to fix it or provide a workaround.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 240, + "prompt_cache_creation_tokens": 208, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37784, + "tokens": 38024 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.txt new file mode 100644 index 000000000..f498893a2 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-auto-hook.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "The result of multiplying 15 by 7 is **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Eq8ECmMIDRgCKkDadtLAudj92ZxoHFr02fRRfLdtYHObOcDiQF6ul57eNMrz7rjJRJ3QrzBGk1NXMBH0GFaqLSi+RlFFxxOIZdVHMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN3AKEriP43sm8rrBRoMQ0btkN0jvb2epKkNIjC7ldUV03HnOSw24KLaHqqU9qwXp/dctQorajSK1JEtAsSaJbi/Vb1CLtVDnYNW9sUq+QJRBOiGixZwhLwkui9WqHYS3vhipPf45xZ/sCcQrEvSC1PTuLt6kOpXnyOxqSAKk/kGlvdzNyrkKJxzUG8y7MlCLVTs3u+JE8pnjUc1PRksWAGYjWpjnTaaVfiozn+OmVC0s9U3SKpk6RMlayil9ZfitdcU1/H6SIE6FMVvCo8YRA5y/G12NMhpkUlqxcnaKZ18dDJNyjoaEVuMF0NZdt93rDhYu6hFl/8RyMbrKTl7GQlniytPbVDlQiirQClhc904iff8vNS6XH7q9NoebDZS76d+OcftyI4PBVjZqZNn203YOMatyfzTgroY34m9HjYVrFoN1vovKeH33T91p3PwxI8mQ65PGHKWd3vPUZ84Hl/Yh9zTtcyIvwBhqOPWoC2/4OA7B65vLt5qPAQKHvWCZQL0m21Xb9A2O0KcW89GJzHm1XpF1YXpWU4DO2K6/yuEYBwU6r3F8P3dTZrv6yfItQWPFOMjD5Bap780cDeDJ12FaLG6mMJdCBgB", + │ │ "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory,\" so I need to use the mcp__calculator__calculator tool to perform this calculation.\n\nLooking at the calculator tool parameters:\n- operation: \"multiply\" (required)\n- a: 15 (required)\n- b: 7 (required)\n\nI'll call the calculator with these parameters.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_cached_tokens": 18783, + │ │ "prompt_tokens": 18793, + │ │ "tokens": 18794 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_01VxxkUqbM8wfd3hj1SmgRLd", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoICCmMIDRgCKkBuOEGYvbAI/nasJ5uoeL24yzLiIeRdwQTt4HJBellK8Aa4yBGKOZWZspHlpdQtB/+d1zkLOcwJby3W7zyAuW/DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCkjTiBiVcs0ZAyYJxoM2JHhZM0wwqkYqmDJIjDR2hf6Tcli+V5NKQaKTlXgE2bdYv/e7PnaemvhIqr8AZ03MCLepewbbvPf6coBZqYqTah0DGLlwMgTGOox+vt8Vp3zXO4bt6BXVTQsWLCAYMVYXMGEEG4u6QhnAt48KBoIMeN3Jh7TpWh/tbmmN14JjUFbkq3cf6+8hWiDXhkEGAE=", + │ "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The result of multiplying 15 by 7 is **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 237, + │ "prompt_cache_creation_tokens": 223, + │ "prompt_cached_tokens": 37566, + │ "prompt_tokens": 37797, + │ "tokens": 38034 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Et8CCmMIDRgCKkC4KFlIDcPjqXKV8EX7ZYp0Oq5ZG2Fl4ESHlmncD4Np+A7nnLohlO8SyZdbcTLy6hhx5h3ZlJp4Q0MXFATUSyTLMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPeXK8/xbZe/mkPmjhoMf/e6FqGNR5+7fYxMIjC8uQkLsE0n0oUNbgnntg//2PV0shjx6lPJ6Kuj5cmUGh06uBaxmHm7IKFgVgFeGZMqqQF6rsHMOKjv7zs0qV1CPLS8rA++ZXCD4QtB58rNrFN1MIscrsGTVRdHanQzC5mhoaKhuMPJQ+YwO5qBrhcTYJw/LE0tsZrkLLRFVu+bn4zbaGQ/mFd3v7joOSXjcNlGse6dORxO1JFWcGIHrqxsi39mwdrJtsS/n1/IlrhABWCgovtZf0nBWAIKSj/CWfZFC7SBG0Tblv7fMeDAsnhEQbiAo8wusLXnT1SxGAE=", + │ │ "thinking": "The user has sent me just \"Part 1\" without any specific task or question. This seems incomplete. I should ask them what they'd like me to help them with.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 161, + │ │ "prompt_cached_tokens": 18650, + │ │ "prompt_tokens": 18660, + │ │ "tokens": 18821 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Eq8DCmMIDRgCKkAsX1krZGggNLkav1y8f1xlySOEtQFt7PJaDzTNcDHCLiNbJHmg1H9X8j8b4dNEema3PvtJvp9QfQXFFX5MvsV/MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHdvoxYPsTAAfYpbgxoMNt2Ae+wHogoCxKxrIjCKTZQ3bl//cEeNG261osG00Jb8/m33Bf+IyKJiTeWn7Am2JtmYlb/pnmki76ayAo0q+QFRk3lTz65m+rnRwVhErO3IbQz7wMC+yV7FXUcrjlijlkcnGTElHar43PwAsTtQJIm1W80eUnoMA4oKslBYknpD5Cwc9dm//MYw5Qtva3jbAj+q+NTDVLla347ok5PBG8XKqaMUkjao/8u7cNkWOfpstBzF8vqOaveFv4msP+tlu1mb3zF1v/2UKXc8pNdojJ/NLbwjNtTHIq/tjvzlaqu77UGkhdcUpmbZK5612upPREcSc79EUcVZFkoxHdt3BPAtccCKENwSlx/OPZ+sedE+0KELr9t/e5jqOZYVxP7pYz7xMAxv2n1D9zO+JwSq4MR0FfAjW4e5PG8YAQ==", + │ "thinking": "The user has now sent \"Part 2\", which still doesn't give me any actual task or question. They seem to be sending numbered messages without context. I should ask them to provide the complete information about what they need help with.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_cache_creation_tokens": 174, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18834, + │ "tokens": 18850 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "ErAECmMIDRgCKkB+s9pCYuRUueJD29vz1evjL5O/fb/4HPxVeQ4GE9uV4spyVVS2MnyioFyJr/nMHesx702uruFoVYv4+zNVFrGtMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOdEIAziK8tIRorTtxoMRci5Re6Cj1+d5RE4IjAPnkmSWVsmJxYeBKm/BEjyJq0IakmSbUJ3U2IrGO5FWv/4z1iTdCweDLgGuZnvHjUq+gLAuEQltv8fB8YqpTPMBIAS237ScFwpTCfhnz/ie7VjEOk2G68I6v3FKi2RYkjtsP4sUDkOAGsaDnek3GP7edOvodB+jdkYhJeSvGGke0Lbli9ZJdWORgNf7aZCZD41enJw3cIgjFOwxlWJHxtmFX+ZEmTkzTXFjlGJWbJwwLwpCiHVMFQEexeuDF5VcEz1TdpTSrdr8rH0OM4Uf/VdMRE4a5SovaJA6TFnLEceyTXpIoG7ct6LyMst2Nt2JvCLPBaYLN46HbuKXZvjF5QU0PSJAeeAJIm2NMmVytew87gR6dR9fjmUhbBKVk4ZStmfILvKqeL9vpDlVKknXFqp5crkbeN9W6aBpZGVhIWKePUs3V7rezhLBrtGzxF+ewxBtoo5xAHVq69llFz+q4R9A4SCBGrJbtHS0SAxM003TCtvt5TaS0rK2zotf4YMAZalWhZm71W1ERVotLdtqhA+4ii6gvKihKHG+hW23xqWQzqvEYEVJzU+jP0HAPcYAQ==", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. They explicitly say not to solve it myself, so I need to use the Agent tool with subagent_type \"math-expert\" and have that agent use the calculator tool to add 15 and 27.\n\nLet me create a task that asks the math-expert agent to add 15 and 27 using the calculator tool.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18811, + │ │ "prompt_tokens": 18821, + │ │ "tokens": 18824 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The result of adding 15 and 27 is 42.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22179, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 127 + │ │ }, + │ │ "cache_creation_input_tokens": 127, + │ │ "cache_read_input_tokens": 22028, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 18, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 using calculator [task] + │ │ input: "Please add 15 and 27 together using the calculator tool. Report the result to me." + │ │ output: [ + │ │ { + │ │ "text": "The result of adding 15 and 27 is 42.", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 using calculator", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + │ │ "claude_agent_sdk.total_tokens": 22037 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cache_creation_tokens": 9148, + │ │ "prompt_cached_tokens": 12880, + │ │ "prompt_tokens": 22031, + │ │ "tokens": 22034 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_01BUZHbp64XJFUZRU2vuUrJ9", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 using calculator", + │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EtwCCmMIDRgCKkCbSWnZ5mk49Ij0H4p1KEDanAwnJP8zG2I8b2HMlpnmrnccGoiO6zt3Kq911LYtcgCeBn5NOIjt+RFZFDKU0UoxMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLuHT/DzOWhhV+Br3RoMoj29xhlCaRtfJM19IjCC7sceqFG4VYeOEiM3vY9FZ+PFe+y+SQbUzf53DefX4SimPWyUgQSQj3X4ovy/NZUqpgGEYmlevBUvBscb6jCJiLOBrz6sHEPCIaS0JlvlEadMNO3E5Y/crefLeaxfF1GRad4rbGWIcbbuhsbk4rQmpA5m1nw2zd2raSfwYqchoPnY2Q2ZNjPAizHngVmMyvt1UowJxsC0LTzGHUpk+uHRdmGMVtTJgJRizi7NhvwmM4q/PYzDttnJSJ69rU5fVb8/CIpgRC3M4+kVcldwC0vA+7vkJxSUgRyBGAE=", + │ "thinking": "Great! The math-expert agent successfully used the calculator tool to add 15 and 27. The result is 42, which is correct. I'll report this to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 315, + │ "prompt_cache_creation_tokens": 318, + │ "prompt_cached_tokens": 37622, + │ "prompt_tokens": 37948, + │ "tokens": 38263 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed the greeting task.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Eu0DCmQIDRgCKkCOpbxDq3fHTCBNR/vFiVrgZg35kTKAwFMsKMpfSRipC241ois8Tank+XxPuWaOSap9mIzdpETFLgexwY4UIStnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgxfoBC8hOxqJzMxeZ8aDOuq83KbtOpng4JqzSIw5+c/dGSrxJsP+a/KIj1hkErEnabsaq72PkvK3qwgYXi2lsDxs+hX4ANhJvFbg94tKrYCCTBXTZqqvUvNnWrEgTPwHhBe09AutUv7sMMOHuc9MXAj/Q7OY7xd0Y5Dni+8R0vdBUV7ykR9S6sYrbidoGsrOIN4/sBX3Z0FsrxT15oZj6K6D3d1yaesCDwFGyNpboLIIkqEav/H7qasuN1RRx4mUrEOJsVMuXY6upMWGzJKSzFTQ/rDnXktl5gmbHy0H0sHwVVUw/1J9S2jRh4g7XAvFEOZp/Zt322xya27TC1cAO31PemCickN/n1F4C+Cvd2UFVNn1PvgvzU0g3fK4WouO9yCf3Vocp/uYrBf0lq9LGrVGhW/vGYUe57n8joHzWyQV4vyVyLsnXJtmy71QENuNP2lCzct3n4mXddRRqJQxdpKRhqA29SzrlWrp/Y5SOWBmbZ0W24Hd5gfi9Zw+2eGgYlJmMFuuRgB", + │ │ "thinking": "The user is instructing me to call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". They're being very explicit that I should not call Bash myself and should not answer with text myself - I must delegate to the echo sub-agent.\n\nLet me follow these instructions exactly.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Please echo a greeting message", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 4, + │ │ "prompt_cached_tokens": 18711, + │ │ "prompt_tokens": 18721, + │ │ "tokens": 18725 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Please echo a greeting message", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please echo a greeting message", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12612, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 83 + │ │ }, + │ │ "cache_creation_input_tokens": 83, + │ │ "cache_read_input_tokens": 12519, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Please echo a greeting message" + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Echo hello message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + │ │ "claude_agent_sdk.total_tokens": 12571 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Please echo a greeting message", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo hello", + │ │ │ "description": "Echo hello message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 49, + │ │ │ "prompt_cache_creation_tokens": 12519, + │ │ │ "prompt_tokens": 12522, + │ │ │ "tokens": 12571 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo hello", + │ │ "description": "Echo hello message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01UePBWB4V5dqHqpS2KiqqXH", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Please echo a greeting message", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo hello", + │ "description": "Echo hello message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EpwGCmQIDRgCKkCksSUYYvmFw41lDV38zAUKG+83X7TBU/YzYmHYMyq5Aq6UbGazCBQbDfQrnvVcmKlQteKdmGESruKJlpZq92vnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgyjH44y9M2K401hdesaDFhA68JEWyW1UKlUQiIwhRlvfLb30g9/+XtIC7a396RgfQAi6xmU6EKoKoUGvG8cdmhcc4Asn+TwW2wr3KXiKuUEmrgQAkBCoDpv9XKMBx+BxuvQ5eqJM1AVBrduqFZzBLhBbs9SAYdSAjXpiI1DP9dVV33FbtLZETurq3rQX90esLk+r+sTEiDxPtzl2EmTUjnHfCYjNbjM/CJRY5mr4hIuinrhqwkG01enWcF/HqReLb6ay4yiqoItBFThpMhthCshRoi1bcp558knW10lHkYvhKG5SsrKbvcKTzklbgKAaqvACyDigdvJ0Me3bt5Q0YI0rKHLWMXZXmLpDOUG/RY8IWg55HBZbsVtWlR1TVQ//o/s1N91Un6oJa5XLGQdzyDiO1iqkPhf5XSejZos69m8tbs0Ee+8mx2l3yOx+hWY3tk5g4MUk9z7Cd95Zt0TAfk8R+hMrX5GPzsoh9bGx+l6SllVptbGIypey4Yt66Cw6wkdNYg2oc4udyrCd27vjjW91MZ63JIxHp75hIfiBBDDrDvuSq7Z3r/K3EDhotwXkOMM0rN78DNsA2jbgeoZmgOOyAE+aRKMVRquHw7IfOMB4TSLFBaY+EaBMf+FjCgWn7SNGo/rJavhHjcU1NrVHxnLnqqxW7eO/A/iT96lGWQ1OpDHZc5FgggBJLQ6ylddjibH58hNXdzTA/2pK7pFgqz1mzKNjBMbJiAZ6jvPDdHZ3d/bSg+QxDeA9V/wLW/75JuVZpD6Tt6qZq9urHXJgqCkN+/kMBw1bui3cJqGSnMvP6LRAqVItnhTvClEEAgYRCyHt93BSIeUPHuIeEJSVk/xKzGGHgYHtlHhKEQylN5bciwwz2pTbzxPmaOQ34Rq8Po2qHkO1L2ZwmE/qaRjRgMpUYHmIRgB", + │ "thinking": "The echo agent has completed. According to the instructions, I was supposed to delegate to the echo sub-agent and not answer with text myself. The agent has completed its task. The system says the result is \"done\" with agent ID ab0746ef3684d1f80.\n\nSince the user instructed me not to answer with text myself and to delegate to the echo sub-agent, and that's what I did, I should probably just confirm that the task was completed or pass along the result. However, the function result doesn't show what the agent actually echoed. Let me just confirm the task was delegated and completed as requested.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed the greeting task.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 268, + │ "prompt_cache_creation_tokens": 246, + │ "prompt_cached_tokens": 37422, + │ "prompt_tokens": 37676, + │ "tokens": 37944 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EpoECmMIDRgCKkC9QeL6AxsMeyrD5cHHu1Tm5rpsHLRj52XKstohEXH8dyOHFIoE5T6D3NMcDG94eGws7KXEmWT30YNQWMP14ks0MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPC5dWnk783Es6mOxhoMEfYMZtnipe97ysV3IjDXPoM0YpVchmgiN64yfoZeaxtvxsJhWRiaSveKSrVLamuhLo1mKHrnl90F72S62c8q5AJWq/fwBQgUjbsuUmjXmhHUORkCgLjzztmhN4YjIiHrt8/p4bIl4f1Ml/C7Fe1gE8s27x1+PJlySQoCgCa9rbhJHJo22S/UAfwMpRFqCV7+kPRwv9JZdMXVGPsAQXihEaiKyt2hp4UDoHpYDDsWxskcKtxcfHc4KS4VPIPTZH6ARIrrrRcpgn/hc6gFf44xiU65+CAX13Iu5hF7yzL24ZOAzjiGiRLhizHMnic2tECaPRoBb6Yi8Noie9Tutv+A3Vlee9R9LbfMhBFunbYYF3S+eCKCrg+SapNkZG7WFD6c+ZnZhyYSJc3yudt1D4syr8kwDaOZ8JIVuC2JA+wKhyujuBvooS3gNNaOdBvgtc4c8sYlLvIGkfUmBAlCTUfcruPuG5lDh9NRRheIDgN5XJUSRNUkiM+RUCrguyMPR6GY+xv+bpszZrbBZxFkIH3T4YTrfi0MIgX7zXQl0X26GvVn6cCwZBgB", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly state \"Do not recover from the error.\" This is asking me to attempt a mathematically invalid operation (division by zero).\n\nI should follow the user's instruction and attempt this operation. Let me call the calculator tool with operation \"divide\", a=2, b=0.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 18784, + │ "prompt_tokens": 18794, + │ "tokens": 18802 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01CMsRz4pP2m3xfVrPNDKi3r", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "EpUDCmMIDRgCKkDR46Ny2JTznygtWJRCCliED4QbRa7fBafCWMqq3Y49sOAvTDXA6T8Lt4iK2G/gn7VDicKOj9yc99O5wtwoG0j7MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOzXlcCXhIuTuTpoZRoM8g8rJArlcfyzlG32IjBPIqicvTdNi18FzFWs8Jq51Tk/AtdOp09pFj8R46e35TtK8kzGGojN6qU3A4Nz8Ikq3wEnAiWddUgsGA9Ku/TeYxPhtL+6ZHW0Dg/S3E9jaKp0LsuoMl+5BvurRRy4FQj46d8h7IrL6pCK6icO8hBH7hPSMwF9vTV9+gHW/06y5vw/qSdsrcbnCiKlQj4nWEU/RFjI6XeFmdhJnSt83O73DsYioFuY49ilOmdzFl5InTqC8gpGsqc6Ml4Fm2sxa8WD+KQq3oVyRdSaI3iNZ02w2IBrkrK5SqvmbuQMF0iFG727fiksW84Mz+RjBBYEEL4FTEf/2P3McWJHVoKiooZdJWBtIu73m7yidEe44nW6E3suGAE=", + "thinking": "The calculator tool returned an error: \"division by zero\". The user explicitly asked me not to recover from the error, so I should just report this error without attempting to fix it or provide a workaround.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 240, + "prompt_cache_creation_tokens": 208, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37784, + "tokens": 38024 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.json new file mode 100644 index 000000000..5d9fff7af --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01VxxkUqbM8wfd3hj1SmgRLd", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq8ECmMIDRgCKkDadtLAudj92ZxoHFr02fRRfLdtYHObOcDiQF6ul57eNMrz7rjJRJ3QrzBGk1NXMBH0GFaqLSi+RlFFxxOIZdVHMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN3AKEriP43sm8rrBRoMQ0btkN0jvb2epKkNIjC7ldUV03HnOSw24KLaHqqU9qwXp/dctQorajSK1JEtAsSaJbi/Vb1CLtVDnYNW9sUq+QJRBOiGixZwhLwkui9WqHYS3vhipPf45xZ/sCcQrEvSC1PTuLt6kOpXnyOxqSAKk/kGlvdzNyrkKJxzUG8y7MlCLVTs3u+JE8pnjUc1PRksWAGYjWpjnTaaVfiozn+OmVC0s9U3SKpk6RMlayil9ZfitdcU1/H6SIE6FMVvCo8YRA5y/G12NMhpkUlqxcnaKZ18dDJNyjoaEVuMF0NZdt93rDhYu6hFl/8RyMbrKTl7GQlniytPbVDlQiirQClhc904iff8vNS6XH7q9NoebDZS76d+OcftyI4PBVjZqZNn203YOMatyfzTgroY34m9HjYVrFoN1vovKeH33T91p3PwxI8mQ65PGHKWd3vPUZ84Hl/Yh9zTtcyIvwBhqOPWoC2/4OA7B65vLt5qPAQKHvWCZQL0m21Xb9A2O0KcW89GJzHm1XpF1YXpWU4DO2K6/yuEYBwU6r3F8P3dTZrv6yfItQWPFOMjD5Bap780cDeDJ12FaLG6mMJdCBgB", + "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory,\" so I need to use the mcp__calculator__calculator tool to perform this calculation.\n\nLooking at the calculator tool parameters:\n- operation: \"multiply\" (required)\n- a: 15 (required)\n- b: 7 (required)\n\nI'll call the calculator with these parameters.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 1, + "prompt_cached_tokens": 18783, + "prompt_tokens": 18793, + "tokens": 18794 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoICCmMIDRgCKkBuOEGYvbAI/nasJ5uoeL24yzLiIeRdwQTt4HJBellK8Aa4yBGKOZWZspHlpdQtB/+d1zkLOcwJby3W7zyAuW/DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCkjTiBiVcs0ZAyYJxoM2JHhZM0wwqkYqmDJIjDR2hf6Tcli+V5NKQaKTlXgE2bdYv/e7PnaemvhIqr8AZ03MCLepewbbvPf6coBZqYqTah0DGLlwMgTGOox+vt8Vp3zXO4bt6BXVTQsWLCAYMVYXMGEEG4u6QhnAt48KBoIMeN3Jh7TpWh/tbmmN14JjUFbkq3cf6+8hWiDXhkEGAE=", + "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The result of multiplying 15 by 7 is **105**.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 237, + "prompt_cache_creation_tokens": 223, + "prompt_cached_tokens": 37566, + "prompt_tokens": 37797, + "tokens": 38034 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "The result of multiplying 15 by 7 is **105**.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Et8CCmMIDRgCKkC4KFlIDcPjqXKV8EX7ZYp0Oq5ZG2Fl4ESHlmncD4Np+A7nnLohlO8SyZdbcTLy6hhx5h3ZlJp4Q0MXFATUSyTLMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPeXK8/xbZe/mkPmjhoMf/e6FqGNR5+7fYxMIjC8uQkLsE0n0oUNbgnntg//2PV0shjx6lPJ6Kuj5cmUGh06uBaxmHm7IKFgVgFeGZMqqQF6rsHMOKjv7zs0qV1CPLS8rA++ZXCD4QtB58rNrFN1MIscrsGTVRdHanQzC5mhoaKhuMPJQ+YwO5qBrhcTYJw/LE0tsZrkLLRFVu+bn4zbaGQ/mFd3v7joOSXjcNlGse6dORxO1JFWcGIHrqxsi39mwdrJtsS/n1/IlrhABWCgovtZf0nBWAIKSj/CWfZFC7SBG0Tblv7fMeDAsnhEQbiAo8wusLXnT1SxGAE=", + "thinking": "The user has sent me just \"Part 1\" without any specific task or question. This seems incomplete. I should ask them what they'd like me to help them with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 161, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18660, + "tokens": 18821 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq8DCmMIDRgCKkAsX1krZGggNLkav1y8f1xlySOEtQFt7PJaDzTNcDHCLiNbJHmg1H9X8j8b4dNEema3PvtJvp9QfQXFFX5MvsV/MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHdvoxYPsTAAfYpbgxoMNt2Ae+wHogoCxKxrIjCKTZQ3bl//cEeNG261osG00Jb8/m33Bf+IyKJiTeWn7Am2JtmYlb/pnmki76ayAo0q+QFRk3lTz65m+rnRwVhErO3IbQz7wMC+yV7FXUcrjlijlkcnGTElHar43PwAsTtQJIm1W80eUnoMA4oKslBYknpD5Cwc9dm//MYw5Qtva3jbAj+q+NTDVLla347ok5PBG8XKqaMUkjao/8u7cNkWOfpstBzF8vqOaveFv4msP+tlu1mb3zF1v/2UKXc8pNdojJ/NLbwjNtTHIq/tjvzlaqu77UGkhdcUpmbZK5612upPREcSc79EUcVZFkoxHdt3BPAtccCKENwSlx/OPZ+sedE+0KELr9t/e5jqOZYVxP7pYz7xMAxv2n1D9zO+JwSq4MR0FfAjW4e5PG8YAQ==", + "thinking": "The user has now sent \"Part 2\", which still doesn't give me any actual task or question. They seem to be sending numbered messages without context. I should ask them to provide the complete information about what they need help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 16, + "prompt_cache_creation_tokens": 174, + "prompt_cached_tokens": 18650, + "prompt_tokens": 18834, + "tokens": 18850 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "ErAECmMIDRgCKkB+s9pCYuRUueJD29vz1evjL5O/fb/4HPxVeQ4GE9uV4spyVVS2MnyioFyJr/nMHesx702uruFoVYv4+zNVFrGtMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOdEIAziK8tIRorTtxoMRci5Re6Cj1+d5RE4IjAPnkmSWVsmJxYeBKm/BEjyJq0IakmSbUJ3U2IrGO5FWv/4z1iTdCweDLgGuZnvHjUq+gLAuEQltv8fB8YqpTPMBIAS237ScFwpTCfhnz/ie7VjEOk2G68I6v3FKi2RYkjtsP4sUDkOAGsaDnek3GP7edOvodB+jdkYhJeSvGGke0Lbli9ZJdWORgNf7aZCZD41enJw3cIgjFOwxlWJHxtmFX+ZEmTkzTXFjlGJWbJwwLwpCiHVMFQEexeuDF5VcEz1TdpTSrdr8rH0OM4Uf/VdMRE4a5SovaJA6TFnLEceyTXpIoG7ct6LyMst2Nt2JvCLPBaYLN46HbuKXZvjF5QU0PSJAeeAJIm2NMmVytew87gR6dR9fjmUhbBKVk4ZStmfILvKqeL9vpDlVKknXFqp5crkbeN9W6aBpZGVhIWKePUs3V7rezhLBrtGzxF+ewxBtoo5xAHVq69llFz+q4R9A4SCBGrJbtHS0SAxM003TCtvt5TaS0rK2zotf4YMAZalWhZm71W1ERVotLdtqhA+4ii6gvKihKHG+hW23xqWQzqvEYEVJzU+jP0HAPcYAQ==", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. They explicitly say not to solve it myself, so I need to use the Agent tool with subagent_type \"math-expert\" and have that agent use the calculator tool to add 15 and 27.\n\nLet me create a task that asks the math-expert agent to add 15 and 27 using the calculator tool.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18811, + "prompt_tokens": 18821, + "tokens": 18824 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 using calculator", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01BUZHbp64XJFUZRU2vuUrJ9", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cache_creation_tokens": 9148, + "prompt_cached_tokens": 12880, + "prompt_tokens": 22031, + "tokens": 22034 + } + } + ], + "input": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "output": [ + { + "text": "The result of adding 15 and 27 is 42.", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 using calculator", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + "claude_agent_sdk.total_tokens": 22037 + } + } + ], + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The result of adding 15 and 27 is 42.", + "type": "text" + } + ], + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22179, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 127 + }, + "cache_creation_input_tokens": 127, + "cache_read_input_tokens": 22028, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 18, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtwCCmMIDRgCKkCbSWnZ5mk49Ij0H4p1KEDanAwnJP8zG2I8b2HMlpnmrnccGoiO6zt3Kq911LYtcgCeBn5NOIjt+RFZFDKU0UoxMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLuHT/DzOWhhV+Br3RoMoj29xhlCaRtfJM19IjCC7sceqFG4VYeOEiM3vY9FZ+PFe+y+SQbUzf53DefX4SimPWyUgQSQj3X4ovy/NZUqpgGEYmlevBUvBscb6jCJiLOBrz6sHEPCIaS0JlvlEadMNO3E5Y/crefLeaxfF1GRad4rbGWIcbbuhsbk4rQmpA5m1nw2zd2raSfwYqchoPnY2Q2ZNjPAizHngVmMyvt1UowJxsC0LTzGHUpk+uHRdmGMVtTJgJRizi7NhvwmM4q/PYzDttnJSJ69rU5fVb8/CIpgRC3M4+kVcldwC0vA+7vkJxSUgRyBGAE=", + "thinking": "Great! The math-expert agent successfully used the calculator tool to add 15 and 27. The result is 42, which is correct. I'll report this to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 315, + "prompt_cache_creation_tokens": 318, + "prompt_cached_tokens": 37622, + "prompt_tokens": 37948, + "tokens": 38263 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eu0DCmQIDRgCKkCOpbxDq3fHTCBNR/vFiVrgZg35kTKAwFMsKMpfSRipC241ois8Tank+XxPuWaOSap9mIzdpETFLgexwY4UIStnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgxfoBC8hOxqJzMxeZ8aDOuq83KbtOpng4JqzSIw5+c/dGSrxJsP+a/KIj1hkErEnabsaq72PkvK3qwgYXi2lsDxs+hX4ANhJvFbg94tKrYCCTBXTZqqvUvNnWrEgTPwHhBe09AutUv7sMMOHuc9MXAj/Q7OY7xd0Y5Dni+8R0vdBUV7ykR9S6sYrbidoGsrOIN4/sBX3Z0FsrxT15oZj6K6D3d1yaesCDwFGyNpboLIIkqEav/H7qasuN1RRx4mUrEOJsVMuXY6upMWGzJKSzFTQ/rDnXktl5gmbHy0H0sHwVVUw/1J9S2jRh4g7XAvFEOZp/Zt322xya27TC1cAO31PemCickN/n1F4C+Cvd2UFVNn1PvgvzU0g3fK4WouO9yCf3Vocp/uYrBf0lq9LGrVGhW/vGYUe57n8joHzWyQV4vyVyLsnXJtmy71QENuNP2lCzct3n4mXddRRqJQxdpKRhqA29SzrlWrp/Y5SOWBmbZ0W24Hd5gfi9Zw+2eGgYlJmMFuuRgB", + "thinking": "The user is instructing me to call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". They're being very explicit that I should not call Bash myself and should not answer with text myself - I must delegate to the echo sub-agent.\n\nLet me follow these instructions exactly.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 18711, + "prompt_tokens": 18721, + "tokens": 18725 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 49, + "prompt_cache_creation_tokens": 12519, + "prompt_tokens": 12522, + "tokens": 12571 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01UePBWB4V5dqHqpS2KiqqXH", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Please echo a greeting message", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Echo hello message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + "claude_agent_sdk.total_tokens": 12571 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Please echo a greeting message", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12612, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 83 + }, + "cache_creation_input_tokens": 83, + "cache_read_input_tokens": 12519, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Please echo a greeting message", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Echo hello message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpwGCmQIDRgCKkCksSUYYvmFw41lDV38zAUKG+83X7TBU/YzYmHYMyq5Aq6UbGazCBQbDfQrnvVcmKlQteKdmGESruKJlpZq92vnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgyjH44y9M2K401hdesaDFhA68JEWyW1UKlUQiIwhRlvfLb30g9/+XtIC7a396RgfQAi6xmU6EKoKoUGvG8cdmhcc4Asn+TwW2wr3KXiKuUEmrgQAkBCoDpv9XKMBx+BxuvQ5eqJM1AVBrduqFZzBLhBbs9SAYdSAjXpiI1DP9dVV33FbtLZETurq3rQX90esLk+r+sTEiDxPtzl2EmTUjnHfCYjNbjM/CJRY5mr4hIuinrhqwkG01enWcF/HqReLb6ay4yiqoItBFThpMhthCshRoi1bcp558knW10lHkYvhKG5SsrKbvcKTzklbgKAaqvACyDigdvJ0Me3bt5Q0YI0rKHLWMXZXmLpDOUG/RY8IWg55HBZbsVtWlR1TVQ//o/s1N91Un6oJa5XLGQdzyDiO1iqkPhf5XSejZos69m8tbs0Ee+8mx2l3yOx+hWY3tk5g4MUk9z7Cd95Zt0TAfk8R+hMrX5GPzsoh9bGx+l6SllVptbGIypey4Yt66Cw6wkdNYg2oc4udyrCd27vjjW91MZ63JIxHp75hIfiBBDDrDvuSq7Z3r/K3EDhotwXkOMM0rN78DNsA2jbgeoZmgOOyAE+aRKMVRquHw7IfOMB4TSLFBaY+EaBMf+FjCgWn7SNGo/rJavhHjcU1NrVHxnLnqqxW7eO/A/iT96lGWQ1OpDHZc5FgggBJLQ6ylddjibH58hNXdzTA/2pK7pFgqz1mzKNjBMbJiAZ6jvPDdHZ3d/bSg+QxDeA9V/wLW/75JuVZpD6Tt6qZq9urHXJgqCkN+/kMBw1bui3cJqGSnMvP6LRAqVItnhTvClEEAgYRCyHt93BSIeUPHuIeEJSVk/xKzGGHgYHtlHhKEQylN5bciwwz2pTbzxPmaOQ34Rq8Po2qHkO1L2ZwmE/qaRjRgMpUYHmIRgB", + "thinking": "The echo agent has completed. According to the instructions, I was supposed to delegate to the echo sub-agent and not answer with text myself. The agent has completed its task. The system says the result is \"done\" with agent ID ab0746ef3684d1f80.\n\nSince the user instructed me not to answer with text myself and to delegate to the echo sub-agent, and that's what I did, I should probably just confirm that the task was completed or pass along the result. However, the function result doesn't show what the agent actually echoed. Let me just confirm the task was delegated and completed as requested.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed the greeting task.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 268, + "prompt_cache_creation_tokens": 246, + "prompt_cached_tokens": 37422, + "prompt_tokens": 37676, + "tokens": 37944 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed the greeting task.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01CMsRz4pP2m3xfVrPNDKi3r", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpoECmMIDRgCKkC9QeL6AxsMeyrD5cHHu1Tm5rpsHLRj52XKstohEXH8dyOHFIoE5T6D3NMcDG94eGws7KXEmWT30YNQWMP14ks0MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPC5dWnk783Es6mOxhoMEfYMZtnipe97ysV3IjDXPoM0YpVchmgiN64yfoZeaxtvxsJhWRiaSveKSrVLamuhLo1mKHrnl90F72S62c8q5AJWq/fwBQgUjbsuUmjXmhHUORkCgLjzztmhN4YjIiHrt8/p4bIl4f1Ml/C7Fe1gE8s27x1+PJlySQoCgCa9rbhJHJo22S/UAfwMpRFqCV7+kPRwv9JZdMXVGPsAQXihEaiKyt2hp4UDoHpYDDsWxskcKtxcfHc4KS4VPIPTZH6ARIrrrRcpgn/hc6gFf44xiU65+CAX13Iu5hF7yzL24ZOAzjiGiRLhizHMnic2tECaPRoBb6Yi8Noie9Tutv+A3Vlee9R9LbfMhBFunbYYF3S+eCKCrg+SapNkZG7WFD6c+ZnZhyYSJc3yudt1D4syr8kwDaOZ8JIVuC2JA+wKhyujuBvooS3gNNaOdBvgtc4c8sYlLvIGkfUmBAlCTUfcruPuG5lDh9NRRheIDgN5XJUSRNUkiM+RUCrguyMPR6GY+xv+bpszZrbBZxFkIH3T4YTrfi0MIgX7zXQl0X26GvVn6cCwZBgB", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly state \"Do not recover from the error.\" This is asking me to attempt a mathematically invalid operation (division by zero).\n\nI should follow the user's instruction and attempt this operation. Let me call the calculator tool with operation \"divide\", a=2, b=0.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 18784, + "prompt_tokens": 18794, + "tokens": 18802 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpUDCmMIDRgCKkDR46Ny2JTznygtWJRCCliED4QbRa7fBafCWMqq3Y49sOAvTDXA6T8Lt4iK2G/gn7VDicKOj9yc99O5wtwoG0j7MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOzXlcCXhIuTuTpoZRoM8g8rJArlcfyzlG32IjBPIqicvTdNi18FzFWs8Jq51Tk/AtdOp09pFj8R46e35TtK8kzGGojN6qU3A4Nz8Ikq3wEnAiWddUgsGA9Ku/TeYxPhtL+6ZHW0Dg/S3E9jaKp0LsuoMl+5BvurRRy4FQj46d8h7IrL6pCK6icO8hBH7hPSMwF9vTV9+gHW/06y5vw/qSdsrcbnCiKlQj4nWEU/RFjI6XeFmdhJnSt83O73DsYioFuY49ilOmdzFl5InTqC8gpGsqc6Ml4Fm2sxa8WD+KQq3oVyRdSaI3iNZ02w2IBrkrK5SqvmbuQMF0iFG727fiksW84Mz+RjBBYEEL4FTEf/2P3McWJHVoKiooZdJWBtIu73m7yidEe44nW6E3suGAE=", + "thinking": "The calculator tool returned an error: \"division by zero\". The user explicitly asked me not to recover from the error, so I should just report this error without attempting to fix it or provide a workaround.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 240, + "prompt_cache_creation_tokens": 208, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37784, + "tokens": 38024 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.txt new file mode 100644 index 000000000..84fc3e813 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76-wrapped.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "The result of multiplying 15 by 7 is **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Eq8ECmMIDRgCKkDadtLAudj92ZxoHFr02fRRfLdtYHObOcDiQF6ul57eNMrz7rjJRJ3QrzBGk1NXMBH0GFaqLSi+RlFFxxOIZdVHMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN3AKEriP43sm8rrBRoMQ0btkN0jvb2epKkNIjC7ldUV03HnOSw24KLaHqqU9qwXp/dctQorajSK1JEtAsSaJbi/Vb1CLtVDnYNW9sUq+QJRBOiGixZwhLwkui9WqHYS3vhipPf45xZ/sCcQrEvSC1PTuLt6kOpXnyOxqSAKk/kGlvdzNyrkKJxzUG8y7MlCLVTs3u+JE8pnjUc1PRksWAGYjWpjnTaaVfiozn+OmVC0s9U3SKpk6RMlayil9ZfitdcU1/H6SIE6FMVvCo8YRA5y/G12NMhpkUlqxcnaKZ18dDJNyjoaEVuMF0NZdt93rDhYu6hFl/8RyMbrKTl7GQlniytPbVDlQiirQClhc904iff8vNS6XH7q9NoebDZS76d+OcftyI4PBVjZqZNn203YOMatyfzTgroY34m9HjYVrFoN1vovKeH33T91p3PwxI8mQ65PGHKWd3vPUZ84Hl/Yh9zTtcyIvwBhqOPWoC2/4OA7B65vLt5qPAQKHvWCZQL0m21Xb9A2O0KcW89GJzHm1XpF1YXpWU4DO2K6/yuEYBwU6r3F8P3dTZrv6yfItQWPFOMjD5Bap780cDeDJ12FaLG6mMJdCBgB", + │ │ "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory,\" so I need to use the mcp__calculator__calculator tool to perform this calculation.\n\nLooking at the calculator tool parameters:\n- operation: \"multiply\" (required)\n- a: 15 (required)\n- b: 7 (required)\n\nI'll call the calculator with these parameters.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_cached_tokens": 18783, + │ │ "prompt_tokens": 18793, + │ │ "tokens": 18794 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_01VxxkUqbM8wfd3hj1SmgRLd", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoICCmMIDRgCKkBuOEGYvbAI/nasJ5uoeL24yzLiIeRdwQTt4HJBellK8Aa4yBGKOZWZspHlpdQtB/+d1zkLOcwJby3W7zyAuW/DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCkjTiBiVcs0ZAyYJxoM2JHhZM0wwqkYqmDJIjDR2hf6Tcli+V5NKQaKTlXgE2bdYv/e7PnaemvhIqr8AZ03MCLepewbbvPf6coBZqYqTah0DGLlwMgTGOox+vt8Vp3zXO4bt6BXVTQsWLCAYMVYXMGEEG4u6QhnAt48KBoIMeN3Jh7TpWh/tbmmN14JjUFbkq3cf6+8hWiDXhkEGAE=", + │ "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The result of multiplying 15 by 7 is **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 237, + │ "prompt_cache_creation_tokens": 223, + │ "prompt_cached_tokens": 37566, + │ "prompt_tokens": 37797, + │ "tokens": 38034 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Et8CCmMIDRgCKkC4KFlIDcPjqXKV8EX7ZYp0Oq5ZG2Fl4ESHlmncD4Np+A7nnLohlO8SyZdbcTLy6hhx5h3ZlJp4Q0MXFATUSyTLMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPeXK8/xbZe/mkPmjhoMf/e6FqGNR5+7fYxMIjC8uQkLsE0n0oUNbgnntg//2PV0shjx6lPJ6Kuj5cmUGh06uBaxmHm7IKFgVgFeGZMqqQF6rsHMOKjv7zs0qV1CPLS8rA++ZXCD4QtB58rNrFN1MIscrsGTVRdHanQzC5mhoaKhuMPJQ+YwO5qBrhcTYJw/LE0tsZrkLLRFVu+bn4zbaGQ/mFd3v7joOSXjcNlGse6dORxO1JFWcGIHrqxsi39mwdrJtsS/n1/IlrhABWCgovtZf0nBWAIKSj/CWfZFC7SBG0Tblv7fMeDAsnhEQbiAo8wusLXnT1SxGAE=", + │ │ "thinking": "The user has sent me just \"Part 1\" without any specific task or question. This seems incomplete. I should ask them what they'd like me to help them with.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 161, + │ │ "prompt_cached_tokens": 18650, + │ │ "prompt_tokens": 18660, + │ │ "tokens": 18821 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I see you've written \"Part 1\" but I don't have the details of what you'd like me to help with. Could you please provide:\n\n1. **What is the task or question?** - What would you like me to do?\n2. **Any relevant context?** - Are you working on a specific project, codebase, or problem?\n3. **What are the expected outcomes?** - What should the result look like?\n\nFeel free to share the full request, and I'll be happy to help!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Eq8DCmMIDRgCKkAsX1krZGggNLkav1y8f1xlySOEtQFt7PJaDzTNcDHCLiNbJHmg1H9X8j8b4dNEema3PvtJvp9QfQXFFX5MvsV/MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHdvoxYPsTAAfYpbgxoMNt2Ae+wHogoCxKxrIjCKTZQ3bl//cEeNG261osG00Jb8/m33Bf+IyKJiTeWn7Am2JtmYlb/pnmki76ayAo0q+QFRk3lTz65m+rnRwVhErO3IbQz7wMC+yV7FXUcrjlijlkcnGTElHar43PwAsTtQJIm1W80eUnoMA4oKslBYknpD5Cwc9dm//MYw5Qtva3jbAj+q+NTDVLla347ok5PBG8XKqaMUkjao/8u7cNkWOfpstBzF8vqOaveFv4msP+tlu1mb3zF1v/2UKXc8pNdojJ/NLbwjNtTHIq/tjvzlaqu77UGkhdcUpmbZK5612upPREcSc79EUcVZFkoxHdt3BPAtccCKENwSlx/OPZ+sedE+0KELr9t/e5jqOZYVxP7pYz7xMAxv2n1D9zO+JwSq4MR0FfAjW4e5PG8YAQ==", + │ "thinking": "The user has now sent \"Part 2\", which still doesn't give me any actual task or question. They seem to be sending numbered messages without context. I should ask them to provide the complete information about what they need help with.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I'm still not seeing the actual task or question! It looks like you're sending numbered parts, but I need the complete request to help you.\n\nCould you please share:\n- **The full task or problem** you'd like help with\n- **Any code, files, or context** related to what you're working on\n- **What you'd like me to do** (e.g., fix a bug, implement a feature, review code, etc.)\n\nFeel free to paste the complete request all at once, and I'll get started! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_cache_creation_tokens": 174, + │ "prompt_cached_tokens": 18650, + │ "prompt_tokens": 18834, + │ "tokens": 18850 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "ErAECmMIDRgCKkB+s9pCYuRUueJD29vz1evjL5O/fb/4HPxVeQ4GE9uV4spyVVS2MnyioFyJr/nMHesx702uruFoVYv4+zNVFrGtMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOdEIAziK8tIRorTtxoMRci5Re6Cj1+d5RE4IjAPnkmSWVsmJxYeBKm/BEjyJq0IakmSbUJ3U2IrGO5FWv/4z1iTdCweDLgGuZnvHjUq+gLAuEQltv8fB8YqpTPMBIAS237ScFwpTCfhnz/ie7VjEOk2G68I6v3FKi2RYkjtsP4sUDkOAGsaDnek3GP7edOvodB+jdkYhJeSvGGke0Lbli9ZJdWORgNf7aZCZD41enJw3cIgjFOwxlWJHxtmFX+ZEmTkzTXFjlGJWbJwwLwpCiHVMFQEexeuDF5VcEz1TdpTSrdr8rH0OM4Uf/VdMRE4a5SovaJA6TFnLEceyTXpIoG7ct6LyMst2Nt2JvCLPBaYLN46HbuKXZvjF5QU0PSJAeeAJIm2NMmVytew87gR6dR9fjmUhbBKVk4ZStmfILvKqeL9vpDlVKknXFqp5crkbeN9W6aBpZGVhIWKePUs3V7rezhLBrtGzxF+ewxBtoo5xAHVq69llFz+q4R9A4SCBGrJbtHS0SAxM003TCtvt5TaS0rK2zotf4YMAZalWhZm71W1ERVotLdtqhA+4ii6gvKihKHG+hW23xqWQzqvEYEVJzU+jP0HAPcYAQ==", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. They explicitly say not to solve it myself, so I need to use the Agent tool with subagent_type \"math-expert\" and have that agent use the calculator tool to add 15 and 27.\n\nLet me create a task that asks the math-expert agent to add 15 and 27 using the calculator tool.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18811, + │ │ "prompt_tokens": 18821, + │ │ "tokens": 18824 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The result of adding 15 and 27 is 42.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22179, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 127 + │ │ }, + │ │ "cache_creation_input_tokens": 127, + │ │ "cache_read_input_tokens": 22028, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 18, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 using calculator [task] + │ │ input: "Please add 15 and 27 together using the calculator tool. Report the result to me." + │ │ output: [ + │ │ { + │ │ "text": "The result of adding 15 and 27 is 42.", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 using calculator", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01F7fpreMTSnWawFUTXi7RgG", + │ │ "claude_agent_sdk.total_tokens": 22037 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cache_creation_tokens": 9148, + │ │ "prompt_cached_tokens": 12880, + │ │ "prompt_tokens": 22031, + │ │ "tokens": 22034 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_01BUZHbp64XJFUZRU2vuUrJ9", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 using calculator", + │ "prompt": "Please add 15 and 27 together using the calculator tool. Report the result to me.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EtwCCmMIDRgCKkCbSWnZ5mk49Ij0H4p1KEDanAwnJP8zG2I8b2HMlpnmrnccGoiO6zt3Kq911LYtcgCeBn5NOIjt+RFZFDKU0UoxMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLuHT/DzOWhhV+Br3RoMoj29xhlCaRtfJM19IjCC7sceqFG4VYeOEiM3vY9FZ+PFe+y+SQbUzf53DefX4SimPWyUgQSQj3X4ovy/NZUqpgGEYmlevBUvBscb6jCJiLOBrz6sHEPCIaS0JlvlEadMNO3E5Y/crefLeaxfF1GRad4rbGWIcbbuhsbk4rQmpA5m1nw2zd2raSfwYqchoPnY2Q2ZNjPAizHngVmMyvt1UowJxsC0LTzGHUpk+uHRdmGMVtTJgJRizi7NhvwmM4q/PYzDttnJSJ69rU5fVb8/CIpgRC3M4+kVcldwC0vA+7vkJxSUgRyBGAE=", + │ "thinking": "Great! The math-expert agent successfully used the calculator tool to add 15 and 27. The result is 42, which is correct. I'll report this to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Perfect! The math-expert subagent has completed the calculation. \n\n**Result: 15 + 27 = 42**\n\nThe agent used the calculator tool to perform the addition operation as requested.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 315, + │ "prompt_cache_creation_tokens": 318, + │ "prompt_cached_tokens": 37622, + │ "prompt_tokens": 37948, + │ "tokens": 38263 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed the greeting task.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "Eu0DCmQIDRgCKkCOpbxDq3fHTCBNR/vFiVrgZg35kTKAwFMsKMpfSRipC241ois8Tank+XxPuWaOSap9mIzdpETFLgexwY4UIStnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgxfoBC8hOxqJzMxeZ8aDOuq83KbtOpng4JqzSIw5+c/dGSrxJsP+a/KIj1hkErEnabsaq72PkvK3qwgYXi2lsDxs+hX4ANhJvFbg94tKrYCCTBXTZqqvUvNnWrEgTPwHhBe09AutUv7sMMOHuc9MXAj/Q7OY7xd0Y5Dni+8R0vdBUV7ykR9S6sYrbidoGsrOIN4/sBX3Z0FsrxT15oZj6K6D3d1yaesCDwFGyNpboLIIkqEav/H7qasuN1RRx4mUrEOJsVMuXY6upMWGzJKSzFTQ/rDnXktl5gmbHy0H0sHwVVUw/1J9S2jRh4g7XAvFEOZp/Zt322xya27TC1cAO31PemCickN/n1F4C+Cvd2UFVNn1PvgvzU0g3fK4WouO9yCf3Vocp/uYrBf0lq9LGrVGhW/vGYUe57n8joHzWyQV4vyVyLsnXJtmy71QENuNP2lCzct3n4mXddRRqJQxdpKRhqA29SzrlWrp/Y5SOWBmbZ0W24Hd5gfi9Zw+2eGgYlJmMFuuRgB", + │ │ "thinking": "The user is instructing me to call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". They're being very explicit that I should not call Bash myself and should not answer with text myself - I must delegate to the echo sub-agent.\n\nLet me follow these instructions exactly.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Please echo a greeting message", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 4, + │ │ "prompt_cached_tokens": 18711, + │ │ "prompt_tokens": 18721, + │ │ "tokens": 18725 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Please echo a greeting message", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please echo a greeting message", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12612, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 83 + │ │ }, + │ │ "cache_creation_input_tokens": 83, + │ │ "cache_read_input_tokens": 12519, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Please echo a greeting message" + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Echo hello message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01GLbf4vFdo9UPmjc9BLacvr", + │ │ "claude_agent_sdk.total_tokens": 12571 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Please echo a greeting message", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo hello", + │ │ │ "description": "Echo hello message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 49, + │ │ │ "prompt_cache_creation_tokens": 12519, + │ │ │ "prompt_tokens": 12522, + │ │ │ "tokens": 12571 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo hello", + │ │ "description": "Echo hello message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01UePBWB4V5dqHqpS2KiqqXH", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Please echo a greeting message", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo hello", + │ "description": "Echo hello message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EpwGCmQIDRgCKkCksSUYYvmFw41lDV38zAUKG+83X7TBU/YzYmHYMyq5Aq6UbGazCBQbDfQrnvVcmKlQteKdmGESruKJlpZq92vnMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgyjH44y9M2K401hdesaDFhA68JEWyW1UKlUQiIwhRlvfLb30g9/+XtIC7a396RgfQAi6xmU6EKoKoUGvG8cdmhcc4Asn+TwW2wr3KXiKuUEmrgQAkBCoDpv9XKMBx+BxuvQ5eqJM1AVBrduqFZzBLhBbs9SAYdSAjXpiI1DP9dVV33FbtLZETurq3rQX90esLk+r+sTEiDxPtzl2EmTUjnHfCYjNbjM/CJRY5mr4hIuinrhqwkG01enWcF/HqReLb6ay4yiqoItBFThpMhthCshRoi1bcp558knW10lHkYvhKG5SsrKbvcKTzklbgKAaqvACyDigdvJ0Me3bt5Q0YI0rKHLWMXZXmLpDOUG/RY8IWg55HBZbsVtWlR1TVQ//o/s1N91Un6oJa5XLGQdzyDiO1iqkPhf5XSejZos69m8tbs0Ee+8mx2l3yOx+hWY3tk5g4MUk9z7Cd95Zt0TAfk8R+hMrX5GPzsoh9bGx+l6SllVptbGIypey4Yt66Cw6wkdNYg2oc4udyrCd27vjjW91MZ63JIxHp75hIfiBBDDrDvuSq7Z3r/K3EDhotwXkOMM0rN78DNsA2jbgeoZmgOOyAE+aRKMVRquHw7IfOMB4TSLFBaY+EaBMf+FjCgWn7SNGo/rJavhHjcU1NrVHxnLnqqxW7eO/A/iT96lGWQ1OpDHZc5FgggBJLQ6ylddjibH58hNXdzTA/2pK7pFgqz1mzKNjBMbJiAZ6jvPDdHZ3d/bSg+QxDeA9V/wLW/75JuVZpD6Tt6qZq9urHXJgqCkN+/kMBw1bui3cJqGSnMvP6LRAqVItnhTvClEEAgYRCyHt93BSIeUPHuIeEJSVk/xKzGGHgYHtlHhKEQylN5bciwwz2pTbzxPmaOQ34Rq8Po2qHkO1L2ZwmE/qaRjRgMpUYHmIRgB", + │ "thinking": "The echo agent has completed. According to the instructions, I was supposed to delegate to the echo sub-agent and not answer with text myself. The agent has completed its task. The system says the result is \"done\" with agent ID ab0746ef3684d1f80.\n\nSince the user instructed me not to answer with text myself and to delegate to the echo sub-agent, and that's what I did, I should probably just confirm that the task was completed or pass along the result. However, the function result doesn't show what the agent actually echoed. Let me just confirm the task was delegated and completed as requested.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed the greeting task.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 268, + │ "prompt_cache_creation_tokens": 246, + │ "prompt_cached_tokens": 37422, + │ "prompt_tokens": 37676, + │ "tokens": 37944 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EpoECmMIDRgCKkC9QeL6AxsMeyrD5cHHu1Tm5rpsHLRj52XKstohEXH8dyOHFIoE5T6D3NMcDG94eGws7KXEmWT30YNQWMP14ks0MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPC5dWnk783Es6mOxhoMEfYMZtnipe97ysV3IjDXPoM0YpVchmgiN64yfoZeaxtvxsJhWRiaSveKSrVLamuhLo1mKHrnl90F72S62c8q5AJWq/fwBQgUjbsuUmjXmhHUORkCgLjzztmhN4YjIiHrt8/p4bIl4f1Ml/C7Fe1gE8s27x1+PJlySQoCgCa9rbhJHJo22S/UAfwMpRFqCV7+kPRwv9JZdMXVGPsAQXihEaiKyt2hp4UDoHpYDDsWxskcKtxcfHc4KS4VPIPTZH6ARIrrrRcpgn/hc6gFf44xiU65+CAX13Iu5hF7yzL24ZOAzjiGiRLhizHMnic2tECaPRoBb6Yi8Noie9Tutv+A3Vlee9R9LbfMhBFunbYYF3S+eCKCrg+SapNkZG7WFD6c+ZnZhyYSJc3yudt1D4syr8kwDaOZ8JIVuC2JA+wKhyujuBvooS3gNNaOdBvgtc4c8sYlLvIGkfUmBAlCTUfcruPuG5lDh9NRRheIDgN5XJUSRNUkiM+RUCrguyMPR6GY+xv+bpszZrbBZxFkIH3T4YTrfi0MIgX7zXQl0X26GvVn6cCwZBgB", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly state \"Do not recover from the error.\" This is asking me to attempt a mathematically invalid operation (division by zero).\n\nI should follow the user's instruction and attempt this operation. Let me call the calculator tool with operation \"divide\", a=2, b=0.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 18784, + │ "prompt_tokens": 18794, + │ "tokens": 18802 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01CMsRz4pP2m3xfVrPNDKi3r", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "EpUDCmMIDRgCKkDR46Ny2JTznygtWJRCCliED4QbRa7fBafCWMqq3Y49sOAvTDXA6T8Lt4iK2G/gn7VDicKOj9yc99O5wtwoG0j7MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOzXlcCXhIuTuTpoZRoM8g8rJArlcfyzlG32IjBPIqicvTdNi18FzFWs8Jq51Tk/AtdOp09pFj8R46e35TtK8kzGGojN6qU3A4Nz8Ikq3wEnAiWddUgsGA9Ku/TeYxPhtL+6ZHW0Dg/S3E9jaKp0LsuoMl+5BvurRRy4FQj46d8h7IrL6pCK6icO8hBH7hPSMwF9vTV9+gHW/06y5vw/qSdsrcbnCiKlQj4nWEU/RFjI6XeFmdhJnSt83O73DsYioFuY49ilOmdzFl5InTqC8gpGsqc6Ml4Fm2sxa8WD+KQq3oVyRdSaI3iNZ02w2IBrkrK5SqvmbuQMF0iFG727fiksW84Mz+RjBBYEEL4FTEf/2P3McWJHVoKiooZdJWBtIu73m7yidEe44nW6E3suGAE=", + "thinking": "The calculator tool returned an error: \"division by zero\". The user explicitly asked me not to recover from the error, so I should just report this error without attempting to fix it or provide a workaround.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The calculator tool returned a division by zero error, as expected for this invalid mathematical operation.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 240, + "prompt_cache_creation_tokens": 208, + "prompt_cached_tokens": 37568, + "prompt_tokens": 37784, + "tokens": 38024 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json deleted file mode 100644 index 5d13ceeaa..000000000 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.76.span-events.json +++ /dev/null @@ -1,316 +0,0 @@ -{ - "async_prompt": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "async-prompt" - }, - "metric_keys": [], - "name": "claude-agent-async-prompt-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "basic": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to multiply 15 by 7. Do not answer from memory." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "basic" - }, - "metric_keys": [], - "name": "claude-agent-basic-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "failure": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to divide 2 by 0. Do not recover from the error." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "failure" - }, - "metric_keys": [], - "name": "claude-agent-failure-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "error": "division by zero", - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "claude-agent-sdk-traces" - }, - "metric_keys": [], - "name": "claude-agent-sdk-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "subagent": { - "handoff_tool": { - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "Agent" - }, - "metric_keys": [], - "name": "tool: Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "nested_task": { - "has_input": false, - "has_output": false, - "metadata": { - "claude_agent_sdk.agent_type": "math-expert", - "claude_agent_sdk.description": "", - "claude_agent_sdk.task_id": "", - "claude_agent_sdk.task_type": "local_agent", - "claude_agent_sdk.tool_use_id": "" - }, - "metric_keys": [], - "name": "Agent: ", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "subagent" - }, - "metric_keys": [], - "name": "claude-agent-subagent-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task_root": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "type": "tool" - } - } -} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.json new file mode 100644 index 000000000..2d6ea7b2e --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_014C2HCQDmrmSYtkyWwretfS", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EroDCmMIDRgCKkCNiw32a3YrAfqx0yCQpG4+1pOEYmjtDLmI5B90sSyISREuifMh274zUTjQIgJ7yeyu5qW9FP0PfdNRwKqRduNaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKbnLnojT8D4Hbf7sBoMtco3I8sm/V6WTR+FIjB299XJH3NOVZ+Mjltpn6ZLKwN3y5lho3VDYA9OtoAFUxYA8ljIC3Yc/ICyG6n4Q9AqhALh6ph2h23Sji3qeR/dsb5UCDq2GCxBQhpEtq+c4iW+5wG9QDn1BMBw2s3DMTEpgqnou03QoVXzVwhcqgkxe5zpVvkEcml9jBvZxQm5vord98W543ATSZScNLUQL3Rpfnlbwc2yQ8u8jwB3PwFyAD1QSfCkxSXIkEcFMo8miGrn6n3q+5ZlwI+8CjQCSxCKbJUsXhCmzTBrmKEupoTU5bUWhARKnKJD+FDUFW59EicHKuqURExgDtWWOj1lp6kx9VUt+O/BSSkNLuhztZlkzG03HorFRn1oXoGuKrE/BKIw6HiZsZpEV7iX5XFMte5rPuITxcfkzXunwhI88zfqHudGsoFSwhgB", + "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. I have the `mcp__calculator__calculator` tool available which can perform arithmetic operations including multiply.\n\nI need to call it with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18871, + "prompt_tokens": 18881, + "tokens": 18884 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EvoBCmMIDRgCKkCYvryhk8U5JayyEzxmOYWnJhTrecNuDNlm9zhADKw8kM6ZDxQ6JIYnrnKAaMaQ+AG1N1tRJW+9xKKL0aKPcbXMMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDIJojfc1OvJvAI3a/BoMuQG42RrO3OEpKoVsIjBuFeyUTS1abDikxFKvP9b+4fwLjqWd389fXiGviqxVYVRkb32bxr6qykOjUI95oxkqRVHjMzweDjKzMszyWEsxAIsWAfxkoEaLbPksvuQMAWOm6+NLaKWbZmhZ2190zdys6CpLiLYLTw3gelCSGPObHnkNtcXrBBgB", + "thinking": "The calculator has returned the result: 15 × 7 = 105", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "15 multiplied by 7 equals **105**.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 201, + "prompt_cache_creation_tokens": 195, + "prompt_cached_tokens": 37742, + "prompt_tokens": 37945, + "tokens": 38146 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "15 multiplied by 7 equals **105**.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoEFCmMIDRgCKkABiax8j/YZAhb/O8Imva6XWP6Y6IahIoOXzjrYIO89qgEwGIq4+znTiA0Xw1nEO/6zOhEvzF6qojeChycY2wuFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFZEvpe8r3PMBJVcIBoM3CE69NMtrhBVQpEQIjA3tT3DEHkBuhSGabcNZBD7c1v2fWlPSTXYUOxdwKTUnNZSkFEdNlh2EghQhdgBcuoqywOeDLx/ZWyyNzD6htzgdakGOgg8vzk4CndWROvM0WQ0lXlDOVdc6izCpLLZHF8z8vTBeIGg37CPutkDbTjD5FEvZcTqIMRwCDrFy83XbGkCBNQWGS9dUHHlbQWpW9uPxfd4zir+/hEjePVXBsbeBiyRBgWAwOf6YnwdssxbkpkMMiFh+XMKmlBXpYY0jSpM2nhADOpaB1Xy8q7UKPyCm31Yin755ZnvexD6PlBHeonkmPEcDJRQq0dQNpeMRRXQKFvz3vjPTt4OjTA1S8ORvwPvaomq6XjcQDNbZW32y4WN8gcK6FOFAoed/E8iB+NTkrJujWjaI0CUa0pDPWvWGb1e+lH2+qRJjUVGYX09Yd8ndpYdsDyArXGAA5slpozgh/KMgi6hwcMi0uC9iCfvxH+hc0yTqm37WMwJcVlHHaZlW8g5jrgJW4RnQmAxLH78ZrwRBmZNpJEI/DZBmZ0hi6LYu0mmn0ZlvDC5bvoYMrr4ylIqH0XeFacMVnOE2oqPx4YMM6kb7e8AeiUBa42mz1jIMUtA5ZhBSe7v9mClzRLHvVRoFC9/Zt8FU09+qTeoEIe37wPZ0GTZ03hITS5WZJeyQBFTFq4kcgRimUsYAQ==", + "thinking": "The user has sent a message that just says \"Part 1\". This is quite vague and doesn't provide enough context for me to understand what they want me to do. \n\nI should ask them to clarify what they need help with. What is \"Part 1\" referring to? Do they want me to:\n- Help with a programming task?\n- Review some code?\n- Start working on a multi-part project?\n- Something else?\n\nI should ask for more information to better understand their request.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 231, + "prompt_cached_tokens": 18738, + "prompt_tokens": 18748, + "tokens": 18979 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoEECmMIDRgCKkBvlSuuH69hfhViLk7nGEq3yT9MWwWhQbZOeAlJ3adq8EjC/GRnDC+Xj9qfzELIzFYK4L4729BHxE+VVg5cyVFrMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDD+966vpxzu9zYWNyxoMBnxZoYQb6TgEPxUpIjAv+SY4+Uec+8cL0GGUL2u57RCg/eMjimvciqNzFy6JhS5rCoOL+JDC9vWQwLqLThsqywINYLJerUA50wRuKTivdN8oXbjI5cvnuaJbf+uUW/ZWGjrXgkLFcEZSeoVGSD3hMP7mwnpEDCumOQnan8OuLRaoKjPJiEu4Xsbp5M/H501ci29bEAUfhwrfx1ll9MdE9tA6Jxq3Mfdwqsnf70M0eid26hvoiNMxH4ANlN/tls/zPtfeuYiz6ZJiwoIIfuq7IskB+2DzvNXaJBosVtAmLfWC31gecA8/DR8EEjrbacJ7jgYcKwRVmhhC2D2Yquuq/KiAXcowZEQA0+v8DEP4Z0+BmW+lFlXinz/QEgJcuJbFphzeB+OwCDz471/1odm92t6V8H5fXEhYVIuTk3MUGwGhD4YG5p+6MHhZdIx/GiJ+agqO50FcPpjuTmyskCor2oHF5lnPcUy1IDp5CcQ26zKRLis/6fSrYsF0a6TmcER5vcT5bbXippFxAqcRGAE=", + "thinking": "The user is now saying \"Part 2\". This is still very vague and doesn't give me any information about what they're working on. They haven't provided any context about what \"Part 1\" and \"Part 2\" refer to.\n\nI should ask them to clarify what they're trying to do. Without more information, I can't help them effectively.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 18, + "prompt_cache_creation_tokens": 244, + "prompt_cached_tokens": 18738, + "prompt_tokens": 18992, + "tokens": 19010 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EqIECmMIDRgCKkCg0DL9Ruqc+/CHaolzSnx/1qgzsUn+kwH9/D2OIcLVbtdxkM7Tm9KyyRj9oeK4+AJNsDO95ULOr0ODgEHbfkaBMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLUGwrDJcrJyJzK2BhoMl6tk+3iG9moyt10HIjBMnkxz8V/cDxAyOC1DPJRuO5YepTZ5Llz3/a7DBDsL53DxNs+s1VR2An82YWOB6DAq7AK3IMrJKtOlPUdQU/HE8wifJzEX4AVXcdyePT15S7BaaJVKXE7ij7BgNYf20QGRN4pabU4LfIIwbeLOMrOHQQZkewt9djLFRKL1hIuXEkltPRYomaghsL3jkmXB6PzTXJS7EikQSDLKs51Gm0WlFaEiuaAx+zcuIDi3iRQH7nPlOevjXyvjJNs7wm1kOh41wOz74yVOJiaYZWExR1CMnzbCsxGe3foovVPYPAHEWZF9eSVjJrUnHdtvJxkQgne/IDnO7J4EU3vqq4Vm3UUs/bczFCNmaQ03L7Xu8yBh0TSW2gKV5fTcVZ5nbgzfF4+6doXi3/JkAv0mX7AtwzyzRqODlXaAjf2urznZeK2l625TG1DQ4kuwUL/SF05slXTORuEagy1r56jl7l5XlwXE3fIK+zPi9oMNQsfmpoafPGBQsgVOim7pnRcCDjZKYNVcdxXE302jPki9Nu+eW9W4xWhww4Kxte4rlyjaBsI1GAE=", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool, and report the result without solving it myself.\n\nI need to use the Agent tool with subagent_type set to \"math-expert\" and give it a task to add 15 and 27 using the calculator tool.\n\nLet me create a clear prompt for the math-expert agent to do this task.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18899, + "prompt_tokens": 18909, + "tokens": 18912 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 using calculator", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_0118HmrDYqo7W6ZHNA7mgGQf", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cache_creation_tokens": 9335, + "prompt_cached_tokens": 12880, + "prompt_tokens": 22218, + "tokens": 22226 + } + } + ], + "input": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "output": [ + { + "text": "The sum of 15 and 27 is **42**.", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 using calculator", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + "claude_agent_sdk.total_tokens": 22234 + } + } + ], + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The sum of 15 and 27 is **42**.", + "type": "text" + } + ], + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22361, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 123 + }, + "cache_creation_input_tokens": 123, + "cache_read_input_tokens": 22215, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 17, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuMCCmMIDRgCKkDQa/mrax7ETIkAw7rU/z63Mu4h/8LNAEuIufr0EnGb1zi2c13AQ042ir362IZVlXZL706L1YTROYg9UifDwcmiMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHEVfGz03megavy9ohoMRKAlne9fjh/R/qMCIjD2CiDqIbcBG8T4w9e49b+7nGg8aRt+4QjyBxv5JVDhs8jFb5HJmdaO9aAUkFe/IPwqrQGsrVkIWx6smLb1tZqmIT0fKkUMkC5F7OVuT3Nb5FX+Y7RXJVD2mNvS18wlN6pD1osZP4TnPQI3yYhbcquQ4momMLIvA5bRZVIhLl8AFgAz1Mtft3w8gKP8yu5ZYdI6GLLGjxT/t/Eb771rAoD3tiT4SQNMi3E/QIvtNvWeMILK3NuYkRMOgl5hTPDCIXiGJ0bBX2D5fyEnRt1yHUZyG6CDQWbGIzFg5xfcxvE23BgB", + "thinking": "Great! The math-expert subagent has completed the task. Using the calculator tool, it determined that 15 + 27 = 42. I can now report this result to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 293, + "prompt_cache_creation_tokens": 326, + "prompt_cached_tokens": 37798, + "prompt_tokens": 38132, + "tokens": 38425 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpAFCmQIDRgCKkAih+NhTHfoockORKkVaC7H3CjuZcE2ALYO5d/Z0k5+rlpm8SUEXBq/wwcQRxu4hYucQORI7Qz5QREVQrzgGUspMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgw9ZExM98GKgAmcVygaDKCnaxVHNhRqDwuXZSIwIVKh7cbjKts8/Eqyd6jJoZeM8yEV5vLRXG9s9+B9Ad2cQ+CGmuYkK2vnfkL+2jlvKtkDVBMTyPVcrz64pIWx2ZDWjV6ACV1l6bnODjOaSLfzxfcYE+zLE+6kBHl2eF9U+xclR6j3HsXQ0zqE7PBaJcB844QcyKe5nnpb9+efdx3NY3A94KpilSJYlnZvTioXsiJI0MXKN7ACEA65g75Cy1je+8pkVlMi9/NNWIHZfP8AKWOvDI0fFmxVqNTi22QPhWi88EACOlDWFFad5cDJj6KmDvkFlRnx1MxKJojkV6xHMG0g/dX4Fcymub/k+LU7Miuf6/Lhn0PKwueVzrRxZap8h+06+X9Yz8SrfC5IRW1Zk5WEvinHPAwTygix/ar2QDRx/mSXiQJ8WSjA8EjdPnUcNKiDyTNzobSsR+Ehwez9euLSGBdfBR5eHyUHcjLM59UPOWs0RMDGUysDY8f4ohRL7GpubBUz5gA57Ux/+rbOaEHLA6zfDYk/1YQrw25z0M5L1qRKnuSyvbzK9WgI+7YWV3fm0NJp5vGis/PmwJHYLqHgKy7cEdJnhF04QZxCxoYMy1WVIse0ptiChJLkaRXq8MJqkm0TsKxEYN7JIJk7gEN+SI5KcX80BI30kAKu9L3OifOzRMLSyVPQTBepCOE6fKNC4OdcQNCt4v+xagNAni+A9JU+izwRsWIYAQ==", + "thinking": "The user is instructing me to call the Agent tool with specific parameters:\n- subagent_type=\"echo\" \n- description=\"echo greeting\"\n\nThey specifically say:\n- Do not call Bash myself\n- Do not answer with text myself\n- Delegate to the echo sub-agent\n\nThe echo agent is described as \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with these parameters and provide a prompt for the echo agent to execute.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 18799, + "prompt_tokens": 18809, + "tokens": 18813 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 65, + "prompt_cache_creation_tokens": 12685, + "prompt_tokens": 12688, + "tokens": 12753 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01A7Epn13kUgYxggsvst9ZjE", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Run an echo command to display a greeting message.", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Display a greeting message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + "claude_agent_sdk.total_tokens": 12753 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Run an echo command to display a greeting message.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12779, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 84 + }, + "cache_creation_input_tokens": 84, + "cache_read_input_tokens": 12685, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Et8ICmQIDRgCKkDvsT9FmICOZF8u3SImgZb7MinP8K/kHvI+8RpCdtmSyrClJ18Jnrs9BpmO8MoVNFk9ufm2VF2oZrS3lBCVqMEuMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgzHgSUStWXKaU7GbN0aDCL4XpLKqy1kjUbV+SIwysbib3qAc3QLw0VHOdycOWc4GdQDFP2On0OOk5GNFvt2hPln+vjsNO1aF0hiQqqDKqgHrDZtnG6KgeZoPQs/OpjY9GsfH7bsFoM12WDjRYKLOf4HQcoWTWLqN++QuADN3KbTKRJ/AFVhHN111Gth8MrNRpFOQUEeFXpQfPSaop16N0p574GLRnnQNNE1l6XYAmC6Z8Xem3ArDJZva4ngU153qeZKZhjeUKcI6eJo7uDdYLT8p/C0LhC/qN+zZusJt8SOFceCBFGbzJxzhjwiN1+xCC/jkl8P0k2qVleg4oHQT28zI4vyhWmWjbLX3AsZZHtSZ4k27BJTm0K8DdbmCPLaaAU9MybqzE4vMrUYQT2BG0LTsYrh6/tMwTKypLFiIKYnvkoj8Td8qrzQ+6PwywANkudsTrpkO8l7C2Dwf6az68lGyi3pKMjwnOHCDz2DyLhm3lb8rRrk/zQA06ZHoN7gNjVevBckom9+1FKkSAv2ck8T69nqgWGS8XVDYWieqGc0ALNRcV6phPPX9osYrxqmC3IoyKI3qRj7ntUi2WyVJ6N7O8t9GZWwSMOHCzWmuk4k41tC3i4gQcNiw47s0gGRGc4h+VodPVecQZKGPVUqcmDupbzhA4TcCQYheFWvoC5Z0N2hiGIw+M/Gfhy9bBV1dpon7DpWZ9e0jeAHfZmaurQ9X+Dtlqp2BdyRC5WTEwDP3/vOF9hgfMEdjr9PS9kWwkiA7kFM5Wh8ju8DrmmvR/lq1Iy4l9vdIyxc6NhbNzuagHHTF6DUAB3LaYotwxR4cImmwKMqoW8s/B+E7Thpzd0izDXWFss2QRWGn2uVSrg2WrTsomCmA8FNYfc0NgII+Xu7t4B14wus+z/2F2S3Q7K6Mo42MukBEaT9YcLwRye1NpuGWla4RhxtsjJf/IyFGz35SF3GwhNFRbu4sip3tMJbVORjM+URn4dtul8F1C/UXfUDSkx49w3Y/vXPPGREwHHPJF8ToTRz6cuGkEKtu/da0ok3LnXWIY7GYlXapFQ7MpMKGy0RbqlVOJtHRC8Egjfvf0SdNYrtYQsSMAopOfyuMXsRVg34iq2w9FjLyvwoSrTPxWrpaVqrJezoISV1wLzq/n8a+SUgpYISOUkrVl9N7QQ1GR0iFPlDN8W/pvQ4jH6fuzRvTXKIrkXoi/fkTj0q8zyXIBMBMA9sjrv0MNvWiHx378uc0wcJ+ZuhUKWQWVr3krrGDTtVUhMRPy7jtDR41zskB6HAhkkKxKq5b3/9VKjQ7K3WMo3yigPqq7YU0J3X5kw7At0s5SZSLnKULccaKf1ldeEjGAE=", + "thinking": "The echo agent has completed. According to the instructions, I should not answer with text myself - I've delegated to the echo sub-agent as requested. The agent has finished its task (status shows \"done\"). \n\nHowever, I should probably inform the user that the agent completed, since the tool description says \"When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nBut the user explicitly said \"Do not answer with text yourself; delegate to the echo sub-agent.\" This seems to be a test instruction. Let me check what the agent actually did by looking at the result - it just says \"done\" with an agent ID.\n\nSince the user said not to answer with text myself, I'll respect that instruction and let the agent's work speak for itself.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed its greeting.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 371, + "prompt_cache_creation_tokens": 313, + "prompt_cached_tokens": 37598, + "prompt_tokens": 37919, + "tokens": 38290 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed its greeting.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01Y5oGGkuKRD8jBgnySf8vwX", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoAECmMIDRgCKkC4P0nCsLhgoAKFqWIb5Cx3b/pf7hAI9lKHDY4H1VldIoqhxPUfQNjYnW/4EEONg2lRZw3VNbW6/0OC5pEKoOvFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOkOmIB+c3J7OC8VKRoM8SC2lSfzYiNQ4C5YIjBM+XL9sYC7No/sopvrceQ6xu8OVQTv2ZC0Sxx1FENpDZsavsXZWY3A6MaLRCkk4AcqygIYJ1LgEQSHSHu09DISTe4s65hFiMEJ6F9/vkpcgab03Tv0gH8p+meaIppb1WA97tiukUiibxJ1UisI25tH1pb7kfxo9h35kewlO0YhofTtuRMZJjb/uwOo/BViK1Ie2QC2GLJxDlKwa+vZ/2/P35QfYi9ZDUggndozbqQEMoPO7JyOqcmgRquL2Uq+FDSzd/IUmVXK+wkB14vpkIfeuXm8D/YmUVpM7pCzRNgn5KKKYc2VXkeo6cM0BX08KzzMfgqKeO/icK0ZeohomldGa2ROdYye0ayW4I/6RNAKdO+VkQ1oEjE0O5G6Gka+Me6UsRQCl++NjXvJuzP2Yp7nPEib8NeFGQon9MaR+0EYDULH1jmQ45mlX3DLOAX1XB/jX030BG+jFp+Ybo0PVWbSARq/Ysy1HoDlluGJgywEQ+IFG76AhnNXaI2CIYoYAQ==", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly say \"Do not recover from the error,\" which means they want me to attempt this operation and let the error happen without trying to catch it or explain it away.\n\nLet me call the calculator tool with operation \"divide\", a=2, and b=0.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 18872, + "prompt_tokens": 18882, + "tokens": 18890 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtcCCmMIDRgCKkB/t1ZHHDAIdUY5gXS4QHCCMRKzeb4FGNLhM5J1LCgMQmZbCbzA33eZgRa2Kx4YiE+Lu61TtHtTLz2RL9St744oMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKzxLGqcjZRl9fPorhoMtE1fy9SIV5OouOd4IjBdxCzinMTOj5zazRCnxw9u1ZQHyjF1FMOvcwLtdpMti4DtQITQX/da82A0Wwf2/moqoQEZceDKMYxwZQFiIlwORqBADwmFjZkC9KCocFBrvS2FLEbaHvd9jheQZJUFj4X7cYGF4cricdmLF5kJ0N+DjKlToBm8FWOkPcgVG7doV52dpUDDJf+5dCn+GyW5uGxhb4PZRtVEEU4Z6V9CbR1GEfE42xwR/hL6qhK0SEtMPr0g5b3UxuDhMl2Yjewxq30VUdK84F9BfyGc7zkUSRhZguNfARgB", + "thinking": "The calculator tool returned an error: \"division by zero\". The user asked me not to recover from the error, so I'll just present the error as-is.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 215, + "prompt_cache_creation_tokens": 205, + "prompt_cached_tokens": 37744, + "prompt_tokens": 37957, + "tokens": 38172 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.txt new file mode 100644 index 000000000..566fa1060 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-auto-hook.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "15 multiplied by 7 equals **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EroDCmMIDRgCKkCNiw32a3YrAfqx0yCQpG4+1pOEYmjtDLmI5B90sSyISREuifMh274zUTjQIgJ7yeyu5qW9FP0PfdNRwKqRduNaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKbnLnojT8D4Hbf7sBoMtco3I8sm/V6WTR+FIjB299XJH3NOVZ+Mjltpn6ZLKwN3y5lho3VDYA9OtoAFUxYA8ljIC3Yc/ICyG6n4Q9AqhALh6ph2h23Sji3qeR/dsb5UCDq2GCxBQhpEtq+c4iW+5wG9QDn1BMBw2s3DMTEpgqnou03QoVXzVwhcqgkxe5zpVvkEcml9jBvZxQm5vord98W543ATSZScNLUQL3Rpfnlbwc2yQ8u8jwB3PwFyAD1QSfCkxSXIkEcFMo8miGrn6n3q+5ZlwI+8CjQCSxCKbJUsXhCmzTBrmKEupoTU5bUWhARKnKJD+FDUFW59EicHKuqURExgDtWWOj1lp6kx9VUt+O/BSSkNLuhztZlkzG03HorFRn1oXoGuKrE/BKIw6HiZsZpEV7iX5XFMte5rPuITxcfkzXunwhI88zfqHudGsoFSwhgB", + │ │ "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. I have the `mcp__calculator__calculator` tool available which can perform arithmetic operations including multiply.\n\nI need to call it with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18871, + │ │ "prompt_tokens": 18881, + │ │ "tokens": 18884 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_014C2HCQDmrmSYtkyWwretfS", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EvoBCmMIDRgCKkCYvryhk8U5JayyEzxmOYWnJhTrecNuDNlm9zhADKw8kM6ZDxQ6JIYnrnKAaMaQ+AG1N1tRJW+9xKKL0aKPcbXMMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDIJojfc1OvJvAI3a/BoMuQG42RrO3OEpKoVsIjBuFeyUTS1abDikxFKvP9b+4fwLjqWd389fXiGviqxVYVRkb32bxr6qykOjUI95oxkqRVHjMzweDjKzMszyWEsxAIsWAfxkoEaLbPksvuQMAWOm6+NLaKWbZmhZ2190zdys6CpLiLYLTw3gelCSGPObHnkNtcXrBBgB", + │ "thinking": "The calculator has returned the result: 15 × 7 = 105", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "15 multiplied by 7 equals **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 201, + │ "prompt_cache_creation_tokens": 195, + │ "prompt_cached_tokens": 37742, + │ "prompt_tokens": 37945, + │ "tokens": 38146 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EoEFCmMIDRgCKkABiax8j/YZAhb/O8Imva6XWP6Y6IahIoOXzjrYIO89qgEwGIq4+znTiA0Xw1nEO/6zOhEvzF6qojeChycY2wuFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFZEvpe8r3PMBJVcIBoM3CE69NMtrhBVQpEQIjA3tT3DEHkBuhSGabcNZBD7c1v2fWlPSTXYUOxdwKTUnNZSkFEdNlh2EghQhdgBcuoqywOeDLx/ZWyyNzD6htzgdakGOgg8vzk4CndWROvM0WQ0lXlDOVdc6izCpLLZHF8z8vTBeIGg37CPutkDbTjD5FEvZcTqIMRwCDrFy83XbGkCBNQWGS9dUHHlbQWpW9uPxfd4zir+/hEjePVXBsbeBiyRBgWAwOf6YnwdssxbkpkMMiFh+XMKmlBXpYY0jSpM2nhADOpaB1Xy8q7UKPyCm31Yin755ZnvexD6PlBHeonkmPEcDJRQq0dQNpeMRRXQKFvz3vjPTt4OjTA1S8ORvwPvaomq6XjcQDNbZW32y4WN8gcK6FOFAoed/E8iB+NTkrJujWjaI0CUa0pDPWvWGb1e+lH2+qRJjUVGYX09Yd8ndpYdsDyArXGAA5slpozgh/KMgi6hwcMi0uC9iCfvxH+hc0yTqm37WMwJcVlHHaZlW8g5jrgJW4RnQmAxLH78ZrwRBmZNpJEI/DZBmZ0hi6LYu0mmn0ZlvDC5bvoYMrr4ylIqH0XeFacMVnOE2oqPx4YMM6kb7e8AeiUBa42mz1jIMUtA5ZhBSe7v9mClzRLHvVRoFC9/Zt8FU09+qTeoEIe37wPZ0GTZ03hITS5WZJeyQBFTFq4kcgRimUsYAQ==", + │ │ "thinking": "The user has sent a message that just says \"Part 1\". This is quite vague and doesn't provide enough context for me to understand what they want me to do. \n\nI should ask them to clarify what they need help with. What is \"Part 1\" referring to? Do they want me to:\n- Help with a programming task?\n- Review some code?\n- Start working on a multi-part project?\n- Something else?\n\nI should ask for more information to better understand their request.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 231, + │ │ "prompt_cached_tokens": 18738, + │ │ "prompt_tokens": 18748, + │ │ "tokens": 18979 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoEECmMIDRgCKkBvlSuuH69hfhViLk7nGEq3yT9MWwWhQbZOeAlJ3adq8EjC/GRnDC+Xj9qfzELIzFYK4L4729BHxE+VVg5cyVFrMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDD+966vpxzu9zYWNyxoMBnxZoYQb6TgEPxUpIjAv+SY4+Uec+8cL0GGUL2u57RCg/eMjimvciqNzFy6JhS5rCoOL+JDC9vWQwLqLThsqywINYLJerUA50wRuKTivdN8oXbjI5cvnuaJbf+uUW/ZWGjrXgkLFcEZSeoVGSD3hMP7mwnpEDCumOQnan8OuLRaoKjPJiEu4Xsbp5M/H501ci29bEAUfhwrfx1ll9MdE9tA6Jxq3Mfdwqsnf70M0eid26hvoiNMxH4ANlN/tls/zPtfeuYiz6ZJiwoIIfuq7IskB+2DzvNXaJBosVtAmLfWC31gecA8/DR8EEjrbacJ7jgYcKwRVmhhC2D2Yquuq/KiAXcowZEQA0+v8DEP4Z0+BmW+lFlXinz/QEgJcuJbFphzeB+OwCDz471/1odm92t6V8H5fXEhYVIuTk3MUGwGhD4YG5p+6MHhZdIx/GiJ+agqO50FcPpjuTmyskCor2oHF5lnPcUy1IDp5CcQ26zKRLis/6fSrYsF0a6TmcER5vcT5bbXippFxAqcRGAE=", + │ "thinking": "The user is now saying \"Part 2\". This is still very vague and doesn't give me any information about what they're working on. They haven't provided any context about what \"Part 1\" and \"Part 2\" refer to.\n\nI should ask them to clarify what they're trying to do. Without more information, I can't help them effectively.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 18, + │ "prompt_cache_creation_tokens": 244, + │ "prompt_cached_tokens": 18738, + │ "prompt_tokens": 18992, + │ "tokens": 19010 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EqIECmMIDRgCKkCg0DL9Ruqc+/CHaolzSnx/1qgzsUn+kwH9/D2OIcLVbtdxkM7Tm9KyyRj9oeK4+AJNsDO95ULOr0ODgEHbfkaBMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLUGwrDJcrJyJzK2BhoMl6tk+3iG9moyt10HIjBMnkxz8V/cDxAyOC1DPJRuO5YepTZ5Llz3/a7DBDsL53DxNs+s1VR2An82YWOB6DAq7AK3IMrJKtOlPUdQU/HE8wifJzEX4AVXcdyePT15S7BaaJVKXE7ij7BgNYf20QGRN4pabU4LfIIwbeLOMrOHQQZkewt9djLFRKL1hIuXEkltPRYomaghsL3jkmXB6PzTXJS7EikQSDLKs51Gm0WlFaEiuaAx+zcuIDi3iRQH7nPlOevjXyvjJNs7wm1kOh41wOz74yVOJiaYZWExR1CMnzbCsxGe3foovVPYPAHEWZF9eSVjJrUnHdtvJxkQgne/IDnO7J4EU3vqq4Vm3UUs/bczFCNmaQ03L7Xu8yBh0TSW2gKV5fTcVZ5nbgzfF4+6doXi3/JkAv0mX7AtwzyzRqODlXaAjf2urznZeK2l625TG1DQ4kuwUL/SF05slXTORuEagy1r56jl7l5XlwXE3fIK+zPi9oMNQsfmpoafPGBQsgVOim7pnRcCDjZKYNVcdxXE302jPki9Nu+eW9W4xWhww4Kxte4rlyjaBsI1GAE=", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool, and report the result without solving it myself.\n\nI need to use the Agent tool with subagent_type set to \"math-expert\" and give it a task to add 15 and 27 using the calculator tool.\n\nLet me create a clear prompt for the math-expert agent to do this task.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18899, + │ │ "prompt_tokens": 18909, + │ │ "tokens": 18912 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The sum of 15 and 27 is **42**.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22361, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 123 + │ │ }, + │ │ "cache_creation_input_tokens": 123, + │ │ "cache_read_input_tokens": 22215, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 17, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 using calculator [task] + │ │ input: "Use the calculator tool to add 15 and 27 together. Report the sum." + │ │ output: [ + │ │ { + │ │ "text": "The sum of 15 and 27 is **42**.", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 using calculator", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + │ │ "claude_agent_sdk.total_tokens": 22234 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cache_creation_tokens": 9335, + │ │ "prompt_cached_tokens": 12880, + │ │ "prompt_tokens": 22218, + │ │ "tokens": 22226 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_0118HmrDYqo7W6ZHNA7mgGQf", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 using calculator", + │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EuMCCmMIDRgCKkDQa/mrax7ETIkAw7rU/z63Mu4h/8LNAEuIufr0EnGb1zi2c13AQ042ir362IZVlXZL706L1YTROYg9UifDwcmiMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHEVfGz03megavy9ohoMRKAlne9fjh/R/qMCIjD2CiDqIbcBG8T4w9e49b+7nGg8aRt+4QjyBxv5JVDhs8jFb5HJmdaO9aAUkFe/IPwqrQGsrVkIWx6smLb1tZqmIT0fKkUMkC5F7OVuT3Nb5FX+Y7RXJVD2mNvS18wlN6pD1osZP4TnPQI3yYhbcquQ4momMLIvA5bRZVIhLl8AFgAz1Mtft3w8gKP8yu5ZYdI6GLLGjxT/t/Eb771rAoD3tiT4SQNMi3E/QIvtNvWeMILK3NuYkRMOgl5hTPDCIXiGJ0bBX2D5fyEnRt1yHUZyG6CDQWbGIzFg5xfcxvE23BgB", + │ "thinking": "Great! The math-expert subagent has completed the task. Using the calculator tool, it determined that 15 + 27 = 42. I can now report this result to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 293, + │ "prompt_cache_creation_tokens": 326, + │ "prompt_cached_tokens": 37798, + │ "prompt_tokens": 38132, + │ "tokens": 38425 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed its greeting.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EpAFCmQIDRgCKkAih+NhTHfoockORKkVaC7H3CjuZcE2ALYO5d/Z0k5+rlpm8SUEXBq/wwcQRxu4hYucQORI7Qz5QREVQrzgGUspMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgw9ZExM98GKgAmcVygaDKCnaxVHNhRqDwuXZSIwIVKh7cbjKts8/Eqyd6jJoZeM8yEV5vLRXG9s9+B9Ad2cQ+CGmuYkK2vnfkL+2jlvKtkDVBMTyPVcrz64pIWx2ZDWjV6ACV1l6bnODjOaSLfzxfcYE+zLE+6kBHl2eF9U+xclR6j3HsXQ0zqE7PBaJcB844QcyKe5nnpb9+efdx3NY3A94KpilSJYlnZvTioXsiJI0MXKN7ACEA65g75Cy1je+8pkVlMi9/NNWIHZfP8AKWOvDI0fFmxVqNTi22QPhWi88EACOlDWFFad5cDJj6KmDvkFlRnx1MxKJojkV6xHMG0g/dX4Fcymub/k+LU7Miuf6/Lhn0PKwueVzrRxZap8h+06+X9Yz8SrfC5IRW1Zk5WEvinHPAwTygix/ar2QDRx/mSXiQJ8WSjA8EjdPnUcNKiDyTNzobSsR+Ehwez9euLSGBdfBR5eHyUHcjLM59UPOWs0RMDGUysDY8f4ohRL7GpubBUz5gA57Ux/+rbOaEHLA6zfDYk/1YQrw25z0M5L1qRKnuSyvbzK9WgI+7YWV3fm0NJp5vGis/PmwJHYLqHgKy7cEdJnhF04QZxCxoYMy1WVIse0ptiChJLkaRXq8MJqkm0TsKxEYN7JIJk7gEN+SI5KcX80BI30kAKu9L3OifOzRMLSyVPQTBepCOE6fKNC4OdcQNCt4v+xagNAni+A9JU+izwRsWIYAQ==", + │ │ "thinking": "The user is instructing me to call the Agent tool with specific parameters:\n- subagent_type=\"echo\" \n- description=\"echo greeting\"\n\nThey specifically say:\n- Do not call Bash myself\n- Do not answer with text myself\n- Delegate to the echo sub-agent\n\nThe echo agent is described as \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with these parameters and provide a prompt for the echo agent to execute.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 4, + │ │ "prompt_cached_tokens": 18799, + │ │ "prompt_tokens": 18809, + │ │ "tokens": 18813 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12779, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 84 + │ │ }, + │ │ "cache_creation_input_tokens": 84, + │ │ "cache_read_input_tokens": 12685, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Run an echo command to display a greeting message." + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Display a greeting message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + │ │ "claude_agent_sdk.total_tokens": 12753 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo hello", + │ │ │ "description": "Display a greeting message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 65, + │ │ │ "prompt_cache_creation_tokens": 12685, + │ │ │ "prompt_tokens": 12688, + │ │ │ "tokens": 12753 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo hello", + │ │ "description": "Display a greeting message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01A7Epn13kUgYxggsvst9ZjE", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Run an echo command to display a greeting message.", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo hello", + │ "description": "Display a greeting message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Et8ICmQIDRgCKkDvsT9FmICOZF8u3SImgZb7MinP8K/kHvI+8RpCdtmSyrClJ18Jnrs9BpmO8MoVNFk9ufm2VF2oZrS3lBCVqMEuMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgzHgSUStWXKaU7GbN0aDCL4XpLKqy1kjUbV+SIwysbib3qAc3QLw0VHOdycOWc4GdQDFP2On0OOk5GNFvt2hPln+vjsNO1aF0hiQqqDKqgHrDZtnG6KgeZoPQs/OpjY9GsfH7bsFoM12WDjRYKLOf4HQcoWTWLqN++QuADN3KbTKRJ/AFVhHN111Gth8MrNRpFOQUEeFXpQfPSaop16N0p574GLRnnQNNE1l6XYAmC6Z8Xem3ArDJZva4ngU153qeZKZhjeUKcI6eJo7uDdYLT8p/C0LhC/qN+zZusJt8SOFceCBFGbzJxzhjwiN1+xCC/jkl8P0k2qVleg4oHQT28zI4vyhWmWjbLX3AsZZHtSZ4k27BJTm0K8DdbmCPLaaAU9MybqzE4vMrUYQT2BG0LTsYrh6/tMwTKypLFiIKYnvkoj8Td8qrzQ+6PwywANkudsTrpkO8l7C2Dwf6az68lGyi3pKMjwnOHCDz2DyLhm3lb8rRrk/zQA06ZHoN7gNjVevBckom9+1FKkSAv2ck8T69nqgWGS8XVDYWieqGc0ALNRcV6phPPX9osYrxqmC3IoyKI3qRj7ntUi2WyVJ6N7O8t9GZWwSMOHCzWmuk4k41tC3i4gQcNiw47s0gGRGc4h+VodPVecQZKGPVUqcmDupbzhA4TcCQYheFWvoC5Z0N2hiGIw+M/Gfhy9bBV1dpon7DpWZ9e0jeAHfZmaurQ9X+Dtlqp2BdyRC5WTEwDP3/vOF9hgfMEdjr9PS9kWwkiA7kFM5Wh8ju8DrmmvR/lq1Iy4l9vdIyxc6NhbNzuagHHTF6DUAB3LaYotwxR4cImmwKMqoW8s/B+E7Thpzd0izDXWFss2QRWGn2uVSrg2WrTsomCmA8FNYfc0NgII+Xu7t4B14wus+z/2F2S3Q7K6Mo42MukBEaT9YcLwRye1NpuGWla4RhxtsjJf/IyFGz35SF3GwhNFRbu4sip3tMJbVORjM+URn4dtul8F1C/UXfUDSkx49w3Y/vXPPGREwHHPJF8ToTRz6cuGkEKtu/da0ok3LnXWIY7GYlXapFQ7MpMKGy0RbqlVOJtHRC8Egjfvf0SdNYrtYQsSMAopOfyuMXsRVg34iq2w9FjLyvwoSrTPxWrpaVqrJezoISV1wLzq/n8a+SUgpYISOUkrVl9N7QQ1GR0iFPlDN8W/pvQ4jH6fuzRvTXKIrkXoi/fkTj0q8zyXIBMBMA9sjrv0MNvWiHx378uc0wcJ+ZuhUKWQWVr3krrGDTtVUhMRPy7jtDR41zskB6HAhkkKxKq5b3/9VKjQ7K3WMo3yigPqq7YU0J3X5kw7At0s5SZSLnKULccaKf1ldeEjGAE=", + │ "thinking": "The echo agent has completed. According to the instructions, I should not answer with text myself - I've delegated to the echo sub-agent as requested. The agent has finished its task (status shows \"done\"). \n\nHowever, I should probably inform the user that the agent completed, since the tool description says \"When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nBut the user explicitly said \"Do not answer with text yourself; delegate to the echo sub-agent.\" This seems to be a test instruction. Let me check what the agent actually did by looking at the result - it just says \"done\" with an agent ID.\n\nSince the user said not to answer with text myself, I'll respect that instruction and let the agent's work speak for itself.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed its greeting.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 371, + │ "prompt_cache_creation_tokens": 313, + │ "prompt_cached_tokens": 37598, + │ "prompt_tokens": 37919, + │ "tokens": 38290 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoAECmMIDRgCKkC4P0nCsLhgoAKFqWIb5Cx3b/pf7hAI9lKHDY4H1VldIoqhxPUfQNjYnW/4EEONg2lRZw3VNbW6/0OC5pEKoOvFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOkOmIB+c3J7OC8VKRoM8SC2lSfzYiNQ4C5YIjBM+XL9sYC7No/sopvrceQ6xu8OVQTv2ZC0Sxx1FENpDZsavsXZWY3A6MaLRCkk4AcqygIYJ1LgEQSHSHu09DISTe4s65hFiMEJ6F9/vkpcgab03Tv0gH8p+meaIppb1WA97tiukUiibxJ1UisI25tH1pb7kfxo9h35kewlO0YhofTtuRMZJjb/uwOo/BViK1Ie2QC2GLJxDlKwa+vZ/2/P35QfYi9ZDUggndozbqQEMoPO7JyOqcmgRquL2Uq+FDSzd/IUmVXK+wkB14vpkIfeuXm8D/YmUVpM7pCzRNgn5KKKYc2VXkeo6cM0BX08KzzMfgqKeO/icK0ZeohomldGa2ROdYye0ayW4I/6RNAKdO+VkQ1oEjE0O5G6Gka+Me6UsRQCl++NjXvJuzP2Yp7nPEib8NeFGQon9MaR+0EYDULH1jmQ45mlX3DLOAX1XB/jX030BG+jFp+Ybo0PVWbSARq/Ysy1HoDlluGJgywEQ+IFG76AhnNXaI2CIYoYAQ==", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly say \"Do not recover from the error,\" which means they want me to attempt this operation and let the error happen without trying to catch it or explain it away.\n\nLet me call the calculator tool with operation \"divide\", a=2, and b=0.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 18872, + │ "prompt_tokens": 18882, + │ "tokens": 18890 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01Y5oGGkuKRD8jBgnySf8vwX", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "EtcCCmMIDRgCKkB/t1ZHHDAIdUY5gXS4QHCCMRKzeb4FGNLhM5J1LCgMQmZbCbzA33eZgRa2Kx4YiE+Lu61TtHtTLz2RL9St744oMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKzxLGqcjZRl9fPorhoMtE1fy9SIV5OouOd4IjBdxCzinMTOj5zazRCnxw9u1ZQHyjF1FMOvcwLtdpMti4DtQITQX/da82A0Wwf2/moqoQEZceDKMYxwZQFiIlwORqBADwmFjZkC9KCocFBrvS2FLEbaHvd9jheQZJUFj4X7cYGF4cricdmLF5kJ0N+DjKlToBm8FWOkPcgVG7doV52dpUDDJf+5dCn+GyW5uGxhb4PZRtVEEU4Z6V9CbR1GEfE42xwR/hL6qhK0SEtMPr0g5b3UxuDhMl2Yjewxq30VUdK84F9BfyGc7zkUSRhZguNfARgB", + "thinking": "The calculator tool returned an error: \"division by zero\". The user asked me not to recover from the error, so I'll just present the error as-is.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 215, + "prompt_cache_creation_tokens": 205, + "prompt_cached_tokens": 37744, + "prompt_tokens": 37957, + "tokens": 38172 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.json new file mode 100644 index 000000000..bb1b94822 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_014C2HCQDmrmSYtkyWwretfS", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EroDCmMIDRgCKkCNiw32a3YrAfqx0yCQpG4+1pOEYmjtDLmI5B90sSyISREuifMh274zUTjQIgJ7yeyu5qW9FP0PfdNRwKqRduNaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKbnLnojT8D4Hbf7sBoMtco3I8sm/V6WTR+FIjB299XJH3NOVZ+Mjltpn6ZLKwN3y5lho3VDYA9OtoAFUxYA8ljIC3Yc/ICyG6n4Q9AqhALh6ph2h23Sji3qeR/dsb5UCDq2GCxBQhpEtq+c4iW+5wG9QDn1BMBw2s3DMTEpgqnou03QoVXzVwhcqgkxe5zpVvkEcml9jBvZxQm5vord98W543ATSZScNLUQL3Rpfnlbwc2yQ8u8jwB3PwFyAD1QSfCkxSXIkEcFMo8miGrn6n3q+5ZlwI+8CjQCSxCKbJUsXhCmzTBrmKEupoTU5bUWhARKnKJD+FDUFW59EicHKuqURExgDtWWOj1lp6kx9VUt+O/BSSkNLuhztZlkzG03HorFRn1oXoGuKrE/BKIw6HiZsZpEV7iX5XFMte5rPuITxcfkzXunwhI88zfqHudGsoFSwhgB", + "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. I have the `mcp__calculator__calculator` tool available which can perform arithmetic operations including multiply.\n\nI need to call it with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18871, + "prompt_tokens": 18881, + "tokens": 18884 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EvoBCmMIDRgCKkCYvryhk8U5JayyEzxmOYWnJhTrecNuDNlm9zhADKw8kM6ZDxQ6JIYnrnKAaMaQ+AG1N1tRJW+9xKKL0aKPcbXMMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDIJojfc1OvJvAI3a/BoMuQG42RrO3OEpKoVsIjBuFeyUTS1abDikxFKvP9b+4fwLjqWd389fXiGviqxVYVRkb32bxr6qykOjUI95oxkqRVHjMzweDjKzMszyWEsxAIsWAfxkoEaLbPksvuQMAWOm6+NLaKWbZmhZ2190zdys6CpLiLYLTw3gelCSGPObHnkNtcXrBBgB", + "thinking": "The calculator has returned the result: 15 × 7 = 105", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "15 multiplied by 7 equals **105**.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 201, + "prompt_cache_creation_tokens": 195, + "prompt_cached_tokens": 37742, + "prompt_tokens": 37945, + "tokens": 38146 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "15 multiplied by 7 equals **105**.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoEFCmMIDRgCKkABiax8j/YZAhb/O8Imva6XWP6Y6IahIoOXzjrYIO89qgEwGIq4+znTiA0Xw1nEO/6zOhEvzF6qojeChycY2wuFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFZEvpe8r3PMBJVcIBoM3CE69NMtrhBVQpEQIjA3tT3DEHkBuhSGabcNZBD7c1v2fWlPSTXYUOxdwKTUnNZSkFEdNlh2EghQhdgBcuoqywOeDLx/ZWyyNzD6htzgdakGOgg8vzk4CndWROvM0WQ0lXlDOVdc6izCpLLZHF8z8vTBeIGg37CPutkDbTjD5FEvZcTqIMRwCDrFy83XbGkCBNQWGS9dUHHlbQWpW9uPxfd4zir+/hEjePVXBsbeBiyRBgWAwOf6YnwdssxbkpkMMiFh+XMKmlBXpYY0jSpM2nhADOpaB1Xy8q7UKPyCm31Yin755ZnvexD6PlBHeonkmPEcDJRQq0dQNpeMRRXQKFvz3vjPTt4OjTA1S8ORvwPvaomq6XjcQDNbZW32y4WN8gcK6FOFAoed/E8iB+NTkrJujWjaI0CUa0pDPWvWGb1e+lH2+qRJjUVGYX09Yd8ndpYdsDyArXGAA5slpozgh/KMgi6hwcMi0uC9iCfvxH+hc0yTqm37WMwJcVlHHaZlW8g5jrgJW4RnQmAxLH78ZrwRBmZNpJEI/DZBmZ0hi6LYu0mmn0ZlvDC5bvoYMrr4ylIqH0XeFacMVnOE2oqPx4YMM6kb7e8AeiUBa42mz1jIMUtA5ZhBSe7v9mClzRLHvVRoFC9/Zt8FU09+qTeoEIe37wPZ0GTZ03hITS5WZJeyQBFTFq4kcgRimUsYAQ==", + "thinking": "The user has sent a message that just says \"Part 1\". This is quite vague and doesn't provide enough context for me to understand what they want me to do. \n\nI should ask them to clarify what they need help with. What is \"Part 1\" referring to? Do they want me to:\n- Help with a programming task?\n- Review some code?\n- Start working on a multi-part project?\n- Something else?\n\nI should ask for more information to better understand their request.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 231, + "prompt_cached_tokens": 18738, + "prompt_tokens": 18748, + "tokens": 18979 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoEECmMIDRgCKkBvlSuuH69hfhViLk7nGEq3yT9MWwWhQbZOeAlJ3adq8EjC/GRnDC+Xj9qfzELIzFYK4L4729BHxE+VVg5cyVFrMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDD+966vpxzu9zYWNyxoMBnxZoYQb6TgEPxUpIjAv+SY4+Uec+8cL0GGUL2u57RCg/eMjimvciqNzFy6JhS5rCoOL+JDC9vWQwLqLThsqywINYLJerUA50wRuKTivdN8oXbjI5cvnuaJbf+uUW/ZWGjrXgkLFcEZSeoVGSD3hMP7mwnpEDCumOQnan8OuLRaoKjPJiEu4Xsbp5M/H501ci29bEAUfhwrfx1ll9MdE9tA6Jxq3Mfdwqsnf70M0eid26hvoiNMxH4ANlN/tls/zPtfeuYiz6ZJiwoIIfuq7IskB+2DzvNXaJBosVtAmLfWC31gecA8/DR8EEjrbacJ7jgYcKwRVmhhC2D2Yquuq/KiAXcowZEQA0+v8DEP4Z0+BmW+lFlXinz/QEgJcuJbFphzeB+OwCDz471/1odm92t6V8H5fXEhYVIuTk3MUGwGhD4YG5p+6MHhZdIx/GiJ+agqO50FcPpjuTmyskCor2oHF5lnPcUy1IDp5CcQ26zKRLis/6fSrYsF0a6TmcER5vcT5bbXippFxAqcRGAE=", + "thinking": "The user is now saying \"Part 2\". This is still very vague and doesn't give me any information about what they're working on. They haven't provided any context about what \"Part 1\" and \"Part 2\" refer to.\n\nI should ask them to clarify what they're trying to do. Without more information, I can't help them effectively.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 18, + "prompt_cache_creation_tokens": 244, + "prompt_cached_tokens": 18738, + "prompt_tokens": 18992, + "tokens": 19010 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EqIECmMIDRgCKkCg0DL9Ruqc+/CHaolzSnx/1qgzsUn+kwH9/D2OIcLVbtdxkM7Tm9KyyRj9oeK4+AJNsDO95ULOr0ODgEHbfkaBMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLUGwrDJcrJyJzK2BhoMl6tk+3iG9moyt10HIjBMnkxz8V/cDxAyOC1DPJRuO5YepTZ5Llz3/a7DBDsL53DxNs+s1VR2An82YWOB6DAq7AK3IMrJKtOlPUdQU/HE8wifJzEX4AVXcdyePT15S7BaaJVKXE7ij7BgNYf20QGRN4pabU4LfIIwbeLOMrOHQQZkewt9djLFRKL1hIuXEkltPRYomaghsL3jkmXB6PzTXJS7EikQSDLKs51Gm0WlFaEiuaAx+zcuIDi3iRQH7nPlOevjXyvjJNs7wm1kOh41wOz74yVOJiaYZWExR1CMnzbCsxGe3foovVPYPAHEWZF9eSVjJrUnHdtvJxkQgne/IDnO7J4EU3vqq4Vm3UUs/bczFCNmaQ03L7Xu8yBh0TSW2gKV5fTcVZ5nbgzfF4+6doXi3/JkAv0mX7AtwzyzRqODlXaAjf2urznZeK2l625TG1DQ4kuwUL/SF05slXTORuEagy1r56jl7l5XlwXE3fIK+zPi9oMNQsfmpoafPGBQsgVOim7pnRcCDjZKYNVcdxXE302jPki9Nu+eW9W4xWhww4Kxte4rlyjaBsI1GAE=", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool, and report the result without solving it myself.\n\nI need to use the Agent tool with subagent_type set to \"math-expert\" and give it a task to add 15 and 27 using the calculator tool.\n\nLet me create a clear prompt for the math-expert agent to do this task.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 18899, + "prompt_tokens": 18909, + "tokens": 18912 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 using calculator", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_0118HmrDYqo7W6ZHNA7mgGQf", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cache_creation_tokens": 9335, + "prompt_cached_tokens": 12880, + "prompt_tokens": 22218, + "tokens": 22226 + } + } + ], + "input": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "output": [ + { + "text": "The sum of 15 and 27 is **42**.", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 using calculator", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + "claude_agent_sdk.total_tokens": 22234 + } + } + ], + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The sum of 15 and 27 is **42**.", + "type": "text" + } + ], + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22361, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 123 + }, + "cache_creation_input_tokens": 123, + "cache_read_input_tokens": 22215, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 17, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 using calculator", + "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuMCCmMIDRgCKkDQa/mrax7ETIkAw7rU/z63Mu4h/8LNAEuIufr0EnGb1zi2c13AQ042ir362IZVlXZL706L1YTROYg9UifDwcmiMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHEVfGz03megavy9ohoMRKAlne9fjh/R/qMCIjD2CiDqIbcBG8T4w9e49b+7nGg8aRt+4QjyBxv5JVDhs8jFb5HJmdaO9aAUkFe/IPwqrQGsrVkIWx6smLb1tZqmIT0fKkUMkC5F7OVuT3Nb5FX+Y7RXJVD2mNvS18wlN6pD1osZP4TnPQI3yYhbcquQ4momMLIvA5bRZVIhLl8AFgAz1Mtft3w8gKP8yu5ZYdI6GLLGjxT/t/Eb771rAoD3tiT4SQNMi3E/QIvtNvWeMILK3NuYkRMOgl5hTPDCIXiGJ0bBX2D5fyEnRt1yHUZyG6CDQWbGIzFg5xfcxvE23BgB", + "thinking": "Great! The math-expert subagent has completed the task. Using the calculator tool, it determined that 15 + 27 = 42. I can now report this result to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 293, + "prompt_cache_creation_tokens": 326, + "prompt_cached_tokens": 37798, + "prompt_tokens": 38132, + "tokens": 38425 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EpAFCmQIDRgCKkAih+NhTHfoockORKkVaC7H3CjuZcE2ALYO5d/Z0k5+rlpm8SUEXBq/wwcQRxu4hYucQORI7Qz5QREVQrzgGUspMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgw9ZExM98GKgAmcVygaDKCnaxVHNhRqDwuXZSIwIVKh7cbjKts8/Eqyd6jJoZeM8yEV5vLRXG9s9+B9Ad2cQ+CGmuYkK2vnfkL+2jlvKtkDVBMTyPVcrz64pIWx2ZDWjV6ACV1l6bnODjOaSLfzxfcYE+zLE+6kBHl2eF9U+xclR6j3HsXQ0zqE7PBaJcB844QcyKe5nnpb9+efdx3NY3A94KpilSJYlnZvTioXsiJI0MXKN7ACEA65g75Cy1je+8pkVlMi9/NNWIHZfP8AKWOvDI0fFmxVqNTi22QPhWi88EACOlDWFFad5cDJj6KmDvkFlRnx1MxKJojkV6xHMG0g/dX4Fcymub/k+LU7Miuf6/Lhn0PKwueVzrRxZap8h+06+X9Yz8SrfC5IRW1Zk5WEvinHPAwTygix/ar2QDRx/mSXiQJ8WSjA8EjdPnUcNKiDyTNzobSsR+Ehwez9euLSGBdfBR5eHyUHcjLM59UPOWs0RMDGUysDY8f4ohRL7GpubBUz5gA57Ux/+rbOaEHLA6zfDYk/1YQrw25z0M5L1qRKnuSyvbzK9WgI+7YWV3fm0NJp5vGis/PmwJHYLqHgKy7cEdJnhF04QZxCxoYMy1WVIse0ptiChJLkaRXq8MJqkm0TsKxEYN7JIJk7gEN+SI5KcX80BI30kAKu9L3OifOzRMLSyVPQTBepCOE6fKNC4OdcQNCt4v+xagNAni+A9JU+izwRsWIYAQ==", + "thinking": "The user is instructing me to call the Agent tool with specific parameters:\n- subagent_type=\"echo\" \n- description=\"echo greeting\"\n\nThey specifically say:\n- Do not call Bash myself\n- Do not answer with text myself\n- Delegate to the echo sub-agent\n\nThe echo agent is described as \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with these parameters and provide a prompt for the echo agent to execute.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 18799, + "prompt_tokens": 18809, + "tokens": 18813 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 65, + "prompt_cache_creation_tokens": 12685, + "prompt_tokens": 12688, + "tokens": 12753 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01A7Epn13kUgYxggsvst9ZjE", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Run an echo command to display a greeting message.", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Display a greeting message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + "claude_agent_sdk.total_tokens": 12753 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Run an echo command to display a greeting message.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12779, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 84 + }, + "cache_creation_input_tokens": 84, + "cache_read_input_tokens": 12685, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run an echo command to display a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo hello", + "description": "Display a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Et8ICmQIDRgCKkDvsT9FmICOZF8u3SImgZb7MinP8K/kHvI+8RpCdtmSyrClJ18Jnrs9BpmO8MoVNFk9ufm2VF2oZrS3lBCVqMEuMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgzHgSUStWXKaU7GbN0aDCL4XpLKqy1kjUbV+SIwysbib3qAc3QLw0VHOdycOWc4GdQDFP2On0OOk5GNFvt2hPln+vjsNO1aF0hiQqqDKqgHrDZtnG6KgeZoPQs/OpjY9GsfH7bsFoM12WDjRYKLOf4HQcoWTWLqN++QuADN3KbTKRJ/AFVhHN111Gth8MrNRpFOQUEeFXpQfPSaop16N0p574GLRnnQNNE1l6XYAmC6Z8Xem3ArDJZva4ngU153qeZKZhjeUKcI6eJo7uDdYLT8p/C0LhC/qN+zZusJt8SOFceCBFGbzJxzhjwiN1+xCC/jkl8P0k2qVleg4oHQT28zI4vyhWmWjbLX3AsZZHtSZ4k27BJTm0K8DdbmCPLaaAU9MybqzE4vMrUYQT2BG0LTsYrh6/tMwTKypLFiIKYnvkoj8Td8qrzQ+6PwywANkudsTrpkO8l7C2Dwf6az68lGyi3pKMjwnOHCDz2DyLhm3lb8rRrk/zQA06ZHoN7gNjVevBckom9+1FKkSAv2ck8T69nqgWGS8XVDYWieqGc0ALNRcV6phPPX9osYrxqmC3IoyKI3qRj7ntUi2WyVJ6N7O8t9GZWwSMOHCzWmuk4k41tC3i4gQcNiw47s0gGRGc4h+VodPVecQZKGPVUqcmDupbzhA4TcCQYheFWvoC5Z0N2hiGIw+M/Gfhy9bBV1dpon7DpWZ9e0jeAHfZmaurQ9X+Dtlqp2BdyRC5WTEwDP3/vOF9hgfMEdjr9PS9kWwkiA7kFM5Wh8ju8DrmmvR/lq1Iy4l9vdIyxc6NhbNzuagHHTF6DUAB3LaYotwxR4cImmwKMqoW8s/B+E7Thpzd0izDXWFss2QRWGn2uVSrg2WrTsomCmA8FNYfc0NgII+Xu7t4B14wus+z/2F2S3Q7K6Mo42MukBEaT9YcLwRye1NpuGWla4RhxtsjJf/IyFGz35SF3GwhNFRbu4sip3tMJbVORjM+URn4dtul8F1C/UXfUDSkx49w3Y/vXPPGREwHHPJF8ToTRz6cuGkEKtu/da0ok3LnXWIY7GYlXapFQ7MpMKGy0RbqlVOJtHRC8Egjfvf0SdNYrtYQsSMAopOfyuMXsRVg34iq2w9FjLyvwoSrTPxWrpaVqrJezoISV1wLzq/n8a+SUgpYISOUkrVl9N7QQ1GR0iFPlDN8W/pvQ4jH6fuzRvTXKIrkXoi/fkTj0q8zyXIBMBMA9sjrv0MNvWiHx378uc0wcJ+ZuhUKWQWVr3krrGDTtVUhMRPy7jtDR41zskB6HAhkkKxKq5b3/9VKjQ7K3WMo3yigPqq7YU0J3X5kw7At0s5SZSLnKULccaKf1ldeEjGAE=", + "thinking": "The echo agent has completed. According to the instructions, I should not answer with text myself - I've delegated to the echo sub-agent as requested. The agent has finished its task (status shows \"done\"). \n\nHowever, I should probably inform the user that the agent completed, since the tool description says \"When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nBut the user explicitly said \"Do not answer with text yourself; delegate to the echo sub-agent.\" This seems to be a test instruction. Let me check what the agent actually did by looking at the result - it just says \"done\" with an agent ID.\n\nSince the user said not to answer with text myself, I'll respect that instruction and let the agent's work speak for itself.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed its greeting.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 371, + "prompt_cache_creation_tokens": 313, + "prompt_cached_tokens": 37598, + "prompt_tokens": 37919, + "tokens": 38290 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed its greeting.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01Y5oGGkuKRD8jBgnySf8vwX", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoAECmMIDRgCKkC4P0nCsLhgoAKFqWIb5Cx3b/pf7hAI9lKHDY4H1VldIoqhxPUfQNjYnW/4EEONg2lRZw3VNbW6/0OC5pEKoOvFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOkOmIB+c3J7OC8VKRoM8SC2lSfzYiNQ4C5YIjBM+XL9sYC7No/sopvrceQ6xu8OVQTv2ZC0Sxx1FENpDZsavsXZWY3A6MaLRCkk4AcqygIYJ1LgEQSHSHu09DISTe4s65hFiMEJ6F9/vkpcgab03Tv0gH8p+meaIppb1WA97tiukUiibxJ1UisI25tH1pb7kfxo9h35kewlO0YhofTtuRMZJjb/uwOo/BViK1Ie2QC2GLJxDlKwa+vZ/2/P35QfYi9ZDUggndozbqQEMoPO7JyOqcmgRquL2Uq+FDSzd/IUmVXK+wkB14vpkIfeuXm8D/YmUVpM7pCzRNgn5KKKYc2VXkeo6cM0BX08KzzMfgqKeO/icK0ZeohomldGa2ROdYye0ayW4I/6RNAKdO+VkQ1oEjE0O5G6Gka+Me6UsRQCl++NjXvJuzP2Yp7nPEib8NeFGQon9MaR+0EYDULH1jmQ45mlX3DLOAX1XB/jX030BG+jFp+Ybo0PVWbSARq/Ysy1HoDlluGJgywEQ+IFG76AhnNXaI2CIYoYAQ==", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly say \"Do not recover from the error,\" which means they want me to attempt this operation and let the error happen without trying to catch it or explain it away.\n\nLet me call the calculator tool with operation \"divide\", a=2, and b=0.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 18872, + "prompt_tokens": 18882, + "tokens": 18890 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtcCCmMIDRgCKkB/t1ZHHDAIdUY5gXS4QHCCMRKzeb4FGNLhM5J1LCgMQmZbCbzA33eZgRa2Kx4YiE+Lu61TtHtTLz2RL9St744oMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKzxLGqcjZRl9fPorhoMtE1fy9SIV5OouOd4IjBdxCzinMTOj5zazRCnxw9u1ZQHyjF1FMOvcwLtdpMti4DtQITQX/da82A0Wwf2/moqoQEZceDKMYxwZQFiIlwORqBADwmFjZkC9KCocFBrvS2FLEbaHvd9jheQZJUFj4X7cYGF4cricdmLF5kJ0N+DjKlToBm8FWOkPcgVG7doV52dpUDDJf+5dCn+GyW5uGxhb4PZRtVEEU4Z6V9CbR1GEfE42xwR/hL6qhK0SEtMPr0g5b3UxuDhMl2Yjewxq30VUdK84F9BfyGc7zkUSRhZguNfARgB", + "thinking": "The calculator tool returned an error: \"division by zero\". The user asked me not to recover from the error, so I'll just present the error as-is.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 215, + "prompt_cache_creation_tokens": 205, + "prompt_cached_tokens": 37744, + "prompt_tokens": 37957, + "tokens": 38172 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.txt new file mode 100644 index 000000000..9dc498ca9 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79-wrapped.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "15 multiplied by 7 equals **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EroDCmMIDRgCKkCNiw32a3YrAfqx0yCQpG4+1pOEYmjtDLmI5B90sSyISREuifMh274zUTjQIgJ7yeyu5qW9FP0PfdNRwKqRduNaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKbnLnojT8D4Hbf7sBoMtco3I8sm/V6WTR+FIjB299XJH3NOVZ+Mjltpn6ZLKwN3y5lho3VDYA9OtoAFUxYA8ljIC3Yc/ICyG6n4Q9AqhALh6ph2h23Sji3qeR/dsb5UCDq2GCxBQhpEtq+c4iW+5wG9QDn1BMBw2s3DMTEpgqnou03QoVXzVwhcqgkxe5zpVvkEcml9jBvZxQm5vord98W543ATSZScNLUQL3Rpfnlbwc2yQ8u8jwB3PwFyAD1QSfCkxSXIkEcFMo8miGrn6n3q+5ZlwI+8CjQCSxCKbJUsXhCmzTBrmKEupoTU5bUWhARKnKJD+FDUFW59EicHKuqURExgDtWWOj1lp6kx9VUt+O/BSSkNLuhztZlkzG03HorFRn1oXoGuKrE/BKIw6HiZsZpEV7iX5XFMte5rPuITxcfkzXunwhI88zfqHudGsoFSwhgB", + │ │ "thinking": "The user wants me to multiply 15 by 7 using the calculator tool. I have the `mcp__calculator__calculator` tool available which can perform arithmetic operations including multiply.\n\nI need to call it with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18871, + │ │ "prompt_tokens": 18881, + │ │ "tokens": 18884 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_014C2HCQDmrmSYtkyWwretfS", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EvoBCmMIDRgCKkCYvryhk8U5JayyEzxmOYWnJhTrecNuDNlm9zhADKw8kM6ZDxQ6JIYnrnKAaMaQ+AG1N1tRJW+9xKKL0aKPcbXMMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDIJojfc1OvJvAI3a/BoMuQG42RrO3OEpKoVsIjBuFeyUTS1abDikxFKvP9b+4fwLjqWd389fXiGviqxVYVRkb32bxr6qykOjUI95oxkqRVHjMzweDjKzMszyWEsxAIsWAfxkoEaLbPksvuQMAWOm6+NLaKWbZmhZ2190zdys6CpLiLYLTw3gelCSGPObHnkNtcXrBBgB", + │ "thinking": "The calculator has returned the result: 15 × 7 = 105", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "15 multiplied by 7 equals **105**.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 201, + │ "prompt_cache_creation_tokens": 195, + │ "prompt_cached_tokens": 37742, + │ "prompt_tokens": 37945, + │ "tokens": 38146 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EoEFCmMIDRgCKkABiax8j/YZAhb/O8Imva6XWP6Y6IahIoOXzjrYIO89qgEwGIq4+znTiA0Xw1nEO/6zOhEvzF6qojeChycY2wuFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFZEvpe8r3PMBJVcIBoM3CE69NMtrhBVQpEQIjA3tT3DEHkBuhSGabcNZBD7c1v2fWlPSTXYUOxdwKTUnNZSkFEdNlh2EghQhdgBcuoqywOeDLx/ZWyyNzD6htzgdakGOgg8vzk4CndWROvM0WQ0lXlDOVdc6izCpLLZHF8z8vTBeIGg37CPutkDbTjD5FEvZcTqIMRwCDrFy83XbGkCBNQWGS9dUHHlbQWpW9uPxfd4zir+/hEjePVXBsbeBiyRBgWAwOf6YnwdssxbkpkMMiFh+XMKmlBXpYY0jSpM2nhADOpaB1Xy8q7UKPyCm31Yin755ZnvexD6PlBHeonkmPEcDJRQq0dQNpeMRRXQKFvz3vjPTt4OjTA1S8ORvwPvaomq6XjcQDNbZW32y4WN8gcK6FOFAoed/E8iB+NTkrJujWjaI0CUa0pDPWvWGb1e+lH2+qRJjUVGYX09Yd8ndpYdsDyArXGAA5slpozgh/KMgi6hwcMi0uC9iCfvxH+hc0yTqm37WMwJcVlHHaZlW8g5jrgJW4RnQmAxLH78ZrwRBmZNpJEI/DZBmZ0hi6LYu0mmn0ZlvDC5bvoYMrr4ylIqH0XeFacMVnOE2oqPx4YMM6kb7e8AeiUBa42mz1jIMUtA5ZhBSe7v9mClzRLHvVRoFC9/Zt8FU09+qTeoEIe37wPZ0GTZ03hITS5WZJeyQBFTFq4kcgRimUsYAQ==", + │ │ "thinking": "The user has sent a message that just says \"Part 1\". This is quite vague and doesn't provide enough context for me to understand what they want me to do. \n\nI should ask them to clarify what they need help with. What is \"Part 1\" referring to? Do they want me to:\n- Help with a programming task?\n- Review some code?\n- Start working on a multi-part project?\n- Something else?\n\nI should ask for more information to better understand their request.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 231, + │ │ "prompt_cached_tokens": 18738, + │ │ "prompt_tokens": 18748, + │ │ "tokens": 18979 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I see you've started with \"Part 1\" - could you provide more context about what you'd like help with? For example:\n\n- Are you working on a multi-part project or task?\n- Do you have code you want me to review or modify?\n- Is there a specific problem you're trying to solve?\n- Do you have files or a codebase you'd like me to explore?\n\nPlease share more details about what you'd like me to work on, and I'll be happy to help!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoEECmMIDRgCKkBvlSuuH69hfhViLk7nGEq3yT9MWwWhQbZOeAlJ3adq8EjC/GRnDC+Xj9qfzELIzFYK4L4729BHxE+VVg5cyVFrMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDD+966vpxzu9zYWNyxoMBnxZoYQb6TgEPxUpIjAv+SY4+Uec+8cL0GGUL2u57RCg/eMjimvciqNzFy6JhS5rCoOL+JDC9vWQwLqLThsqywINYLJerUA50wRuKTivdN8oXbjI5cvnuaJbf+uUW/ZWGjrXgkLFcEZSeoVGSD3hMP7mwnpEDCumOQnan8OuLRaoKjPJiEu4Xsbp5M/H501ci29bEAUfhwrfx1ll9MdE9tA6Jxq3Mfdwqsnf70M0eid26hvoiNMxH4ANlN/tls/zPtfeuYiz6ZJiwoIIfuq7IskB+2DzvNXaJBosVtAmLfWC31gecA8/DR8EEjrbacJ7jgYcKwRVmhhC2D2Yquuq/KiAXcowZEQA0+v8DEP4Z0+BmW+lFlXinz/QEgJcuJbFphzeB+OwCDz471/1odm92t6V8H5fXEhYVIuTk3MUGwGhD4YG5p+6MHhZdIx/GiJ+agqO50FcPpjuTmyskCor2oHF5lnPcUy1IDp5CcQ26zKRLis/6fSrYsF0a6TmcER5vcT5bbXippFxAqcRGAE=", + │ "thinking": "The user is now saying \"Part 2\". This is still very vague and doesn't give me any information about what they're working on. They haven't provided any context about what \"Part 1\" and \"Part 2\" refer to.\n\nI should ask them to clarify what they're trying to do. Without more information, I can't help them effectively.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I still need more information to help you! You've mentioned \"Part 1\" and \"Part 2,\" but I don't have context about what you're working on.\n\nCould you please provide:\n\n1. **What is the overall task or project?** (e.g., \"Building a web app,\" \"Fixing a bug,\" \"Learning a new technology\")\n2. **What should Part 1 do?** What code or files should I work with?\n3. **What should Part 2 do?** What's the next step?\n4. **Do you have existing code** you want me to look at, or should I start from scratch?\n\nOnce you give me these details, I'll be able to help you effectively!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 18, + │ "prompt_cache_creation_tokens": 244, + │ "prompt_cached_tokens": 18738, + │ "prompt_tokens": 18992, + │ "tokens": 19010 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EqIECmMIDRgCKkCg0DL9Ruqc+/CHaolzSnx/1qgzsUn+kwH9/D2OIcLVbtdxkM7Tm9KyyRj9oeK4+AJNsDO95ULOr0ODgEHbfkaBMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLUGwrDJcrJyJzK2BhoMl6tk+3iG9moyt10HIjBMnkxz8V/cDxAyOC1DPJRuO5YepTZ5Llz3/a7DBDsL53DxNs+s1VR2An82YWOB6DAq7AK3IMrJKtOlPUdQU/HE8wifJzEX4AVXcdyePT15S7BaaJVKXE7ij7BgNYf20QGRN4pabU4LfIIwbeLOMrOHQQZkewt9djLFRKL1hIuXEkltPRYomaghsL3jkmXB6PzTXJS7EikQSDLKs51Gm0WlFaEiuaAx+zcuIDi3iRQH7nPlOevjXyvjJNs7wm1kOh41wOz74yVOJiaYZWExR1CMnzbCsxGe3foovVPYPAHEWZF9eSVjJrUnHdtvJxkQgne/IDnO7J4EU3vqq4Vm3UUs/bczFCNmaQ03L7Xu8yBh0TSW2gKV5fTcVZ5nbgzfF4+6doXi3/JkAv0mX7AtwzyzRqODlXaAjf2urznZeK2l625TG1DQ4kuwUL/SF05slXTORuEagy1r56jl7l5XlwXE3fIK+zPi9oMNQsfmpoafPGBQsgVOim7pnRcCDjZKYNVcdxXE302jPki9Nu+eW9W4xWhww4Kxte4rlyjaBsI1GAE=", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool, and report the result without solving it myself.\n\nI need to use the Agent tool with subagent_type set to \"math-expert\" and give it a task to add 15 and 27 using the calculator tool.\n\nLet me create a clear prompt for the math-expert agent to do this task.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 18899, + │ │ "prompt_tokens": 18909, + │ │ "tokens": 18912 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The sum of 15 and 27 is **42**.", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22361, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 123 + │ │ }, + │ │ "cache_creation_input_tokens": 123, + │ │ "cache_read_input_tokens": 22215, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 17, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 using calculator [task] + │ │ input: "Use the calculator tool to add 15 and 27 together. Report the sum." + │ │ output: [ + │ │ { + │ │ "text": "The sum of 15 and 27 is **42**.", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 using calculator", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01WQPJ1E4tonQtywdoKhy5Bh", + │ │ "claude_agent_sdk.total_tokens": 22234 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 using calculator", + │ │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cache_creation_tokens": 9335, + │ │ "prompt_cached_tokens": 12880, + │ │ "prompt_tokens": 22218, + │ │ "tokens": 22226 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_0118HmrDYqo7W6ZHNA7mgGQf", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 using calculator", + │ "prompt": "Use the calculator tool to add 15 and 27 together. Report the sum.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EuMCCmMIDRgCKkDQa/mrax7ETIkAw7rU/z63Mu4h/8LNAEuIufr0EnGb1zi2c13AQ042ir362IZVlXZL706L1YTROYg9UifDwcmiMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDHEVfGz03megavy9ohoMRKAlne9fjh/R/qMCIjD2CiDqIbcBG8T4w9e49b+7nGg8aRt+4QjyBxv5JVDhs8jFb5HJmdaO9aAUkFe/IPwqrQGsrVkIWx6smLb1tZqmIT0fKkUMkC5F7OVuT3Nb5FX+Y7RXJVD2mNvS18wlN6pD1osZP4TnPQI3yYhbcquQ4momMLIvA5bRZVIhLl8AFgAz1Mtft3w8gKP8yu5ZYdI6GLLGjxT/t/Eb771rAoD3tiT4SQNMi3E/QIvtNvWeMILK3NuYkRMOgl5hTPDCIXiGJ0bBX2D5fyEnRt1yHUZyG6CDQWbGIzFg5xfcxvE23BgB", + │ "thinking": "Great! The math-expert subagent has completed the task. Using the calculator tool, it determined that 15 + 27 = 42. I can now report this result to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 293, + │ "prompt_cache_creation_tokens": 326, + │ "prompt_cached_tokens": 37798, + │ "prompt_tokens": 38132, + │ "tokens": 38425 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed its greeting.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EpAFCmQIDRgCKkAih+NhTHfoockORKkVaC7H3CjuZcE2ALYO5d/Z0k5+rlpm8SUEXBq/wwcQRxu4hYucQORI7Qz5QREVQrzgGUspMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgw9ZExM98GKgAmcVygaDKCnaxVHNhRqDwuXZSIwIVKh7cbjKts8/Eqyd6jJoZeM8yEV5vLRXG9s9+B9Ad2cQ+CGmuYkK2vnfkL+2jlvKtkDVBMTyPVcrz64pIWx2ZDWjV6ACV1l6bnODjOaSLfzxfcYE+zLE+6kBHl2eF9U+xclR6j3HsXQ0zqE7PBaJcB844QcyKe5nnpb9+efdx3NY3A94KpilSJYlnZvTioXsiJI0MXKN7ACEA65g75Cy1je+8pkVlMi9/NNWIHZfP8AKWOvDI0fFmxVqNTi22QPhWi88EACOlDWFFad5cDJj6KmDvkFlRnx1MxKJojkV6xHMG0g/dX4Fcymub/k+LU7Miuf6/Lhn0PKwueVzrRxZap8h+06+X9Yz8SrfC5IRW1Zk5WEvinHPAwTygix/ar2QDRx/mSXiQJ8WSjA8EjdPnUcNKiDyTNzobSsR+Ehwez9euLSGBdfBR5eHyUHcjLM59UPOWs0RMDGUysDY8f4ohRL7GpubBUz5gA57Ux/+rbOaEHLA6zfDYk/1YQrw25z0M5L1qRKnuSyvbzK9WgI+7YWV3fm0NJp5vGis/PmwJHYLqHgKy7cEdJnhF04QZxCxoYMy1WVIse0ptiChJLkaRXq8MJqkm0TsKxEYN7JIJk7gEN+SI5KcX80BI30kAKu9L3OifOzRMLSyVPQTBepCOE6fKNC4OdcQNCt4v+xagNAni+A9JU+izwRsWIYAQ==", + │ │ "thinking": "The user is instructing me to call the Agent tool with specific parameters:\n- subagent_type=\"echo\" \n- description=\"echo greeting\"\n\nThey specifically say:\n- Do not call Bash myself\n- Do not answer with text myself\n- Delegate to the echo sub-agent\n\nThe echo agent is described as \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with these parameters and provide a prompt for the echo agent to execute.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 4, + │ │ "prompt_cached_tokens": 18799, + │ │ "prompt_tokens": 18809, + │ │ "tokens": 18813 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12779, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 84 + │ │ }, + │ │ "cache_creation_input_tokens": 84, + │ │ "cache_read_input_tokens": 12685, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Run an echo command to display a greeting message." + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Display a greeting message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01BtA1cWsQzFkzg6pfASfPmK", + │ │ "claude_agent_sdk.total_tokens": 12753 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Run an echo command to display a greeting message.", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo hello", + │ │ │ "description": "Display a greeting message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 65, + │ │ │ "prompt_cache_creation_tokens": 12685, + │ │ │ "prompt_tokens": 12688, + │ │ │ "tokens": 12753 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo hello", + │ │ "description": "Display a greeting message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01A7Epn13kUgYxggsvst9ZjE", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Run an echo command to display a greeting message.", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo hello", + │ "description": "Display a greeting message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Et8ICmQIDRgCKkDvsT9FmICOZF8u3SImgZb7MinP8K/kHvI+8RpCdtmSyrClJ18Jnrs9BpmO8MoVNFk9ufm2VF2oZrS3lBCVqMEuMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgzHgSUStWXKaU7GbN0aDCL4XpLKqy1kjUbV+SIwysbib3qAc3QLw0VHOdycOWc4GdQDFP2On0OOk5GNFvt2hPln+vjsNO1aF0hiQqqDKqgHrDZtnG6KgeZoPQs/OpjY9GsfH7bsFoM12WDjRYKLOf4HQcoWTWLqN++QuADN3KbTKRJ/AFVhHN111Gth8MrNRpFOQUEeFXpQfPSaop16N0p574GLRnnQNNE1l6XYAmC6Z8Xem3ArDJZva4ngU153qeZKZhjeUKcI6eJo7uDdYLT8p/C0LhC/qN+zZusJt8SOFceCBFGbzJxzhjwiN1+xCC/jkl8P0k2qVleg4oHQT28zI4vyhWmWjbLX3AsZZHtSZ4k27BJTm0K8DdbmCPLaaAU9MybqzE4vMrUYQT2BG0LTsYrh6/tMwTKypLFiIKYnvkoj8Td8qrzQ+6PwywANkudsTrpkO8l7C2Dwf6az68lGyi3pKMjwnOHCDz2DyLhm3lb8rRrk/zQA06ZHoN7gNjVevBckom9+1FKkSAv2ck8T69nqgWGS8XVDYWieqGc0ALNRcV6phPPX9osYrxqmC3IoyKI3qRj7ntUi2WyVJ6N7O8t9GZWwSMOHCzWmuk4k41tC3i4gQcNiw47s0gGRGc4h+VodPVecQZKGPVUqcmDupbzhA4TcCQYheFWvoC5Z0N2hiGIw+M/Gfhy9bBV1dpon7DpWZ9e0jeAHfZmaurQ9X+Dtlqp2BdyRC5WTEwDP3/vOF9hgfMEdjr9PS9kWwkiA7kFM5Wh8ju8DrmmvR/lq1Iy4l9vdIyxc6NhbNzuagHHTF6DUAB3LaYotwxR4cImmwKMqoW8s/B+E7Thpzd0izDXWFss2QRWGn2uVSrg2WrTsomCmA8FNYfc0NgII+Xu7t4B14wus+z/2F2S3Q7K6Mo42MukBEaT9YcLwRye1NpuGWla4RhxtsjJf/IyFGz35SF3GwhNFRbu4sip3tMJbVORjM+URn4dtul8F1C/UXfUDSkx49w3Y/vXPPGREwHHPJF8ToTRz6cuGkEKtu/da0ok3LnXWIY7GYlXapFQ7MpMKGy0RbqlVOJtHRC8Egjfvf0SdNYrtYQsSMAopOfyuMXsRVg34iq2w9FjLyvwoSrTPxWrpaVqrJezoISV1wLzq/n8a+SUgpYISOUkrVl9N7QQ1GR0iFPlDN8W/pvQ4jH6fuzRvTXKIrkXoi/fkTj0q8zyXIBMBMA9sjrv0MNvWiHx378uc0wcJ+ZuhUKWQWVr3krrGDTtVUhMRPy7jtDR41zskB6HAhkkKxKq5b3/9VKjQ7K3WMo3yigPqq7YU0J3X5kw7At0s5SZSLnKULccaKf1ldeEjGAE=", + │ "thinking": "The echo agent has completed. According to the instructions, I should not answer with text myself - I've delegated to the echo sub-agent as requested. The agent has finished its task (status shows \"done\"). \n\nHowever, I should probably inform the user that the agent completed, since the tool description says \"When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nBut the user explicitly said \"Do not answer with text yourself; delegate to the echo sub-agent.\" This seems to be a test instruction. Let me check what the agent actually did by looking at the result - it just says \"done\" with an agent ID.\n\nSince the user said not to answer with text myself, I'll respect that instruction and let the agent's work speak for itself.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed its greeting.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 371, + │ "prompt_cache_creation_tokens": 313, + │ "prompt_cached_tokens": 37598, + │ "prompt_tokens": 37919, + │ "tokens": 38290 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoAECmMIDRgCKkC4P0nCsLhgoAKFqWIb5Cx3b/pf7hAI9lKHDY4H1VldIoqhxPUfQNjYnW/4EEONg2lRZw3VNbW6/0OC5pEKoOvFMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDOkOmIB+c3J7OC8VKRoM8SC2lSfzYiNQ4C5YIjBM+XL9sYC7No/sopvrceQ6xu8OVQTv2ZC0Sxx1FENpDZsavsXZWY3A6MaLRCkk4AcqygIYJ1LgEQSHSHu09DISTe4s65hFiMEJ6F9/vkpcgab03Tv0gH8p+meaIppb1WA97tiukUiibxJ1UisI25tH1pb7kfxo9h35kewlO0YhofTtuRMZJjb/uwOo/BViK1Ie2QC2GLJxDlKwa+vZ/2/P35QfYi9ZDUggndozbqQEMoPO7JyOqcmgRquL2Uq+FDSzd/IUmVXK+wkB14vpkIfeuXm8D/YmUVpM7pCzRNgn5KKKYc2VXkeo6cM0BX08KzzMfgqKeO/icK0ZeohomldGa2ROdYye0ayW4I/6RNAKdO+VkQ1oEjE0O5G6Gka+Me6UsRQCl++NjXvJuzP2Yp7nPEib8NeFGQon9MaR+0EYDULH1jmQ45mlX3DLOAX1XB/jX030BG+jFp+Ybo0PVWbSARq/Ysy1HoDlluGJgywEQ+IFG76AhnNXaI2CIYoYAQ==", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0. They explicitly say \"Do not recover from the error,\" which means they want me to attempt this operation and let the error happen without trying to catch it or explain it away.\n\nLet me call the calculator tool with operation \"divide\", a=2, and b=0.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 18872, + │ "prompt_tokens": 18882, + │ "tokens": 18890 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01Y5oGGkuKRD8jBgnySf8vwX", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "EtcCCmMIDRgCKkB/t1ZHHDAIdUY5gXS4QHCCMRKzeb4FGNLhM5J1LCgMQmZbCbzA33eZgRa2Kx4YiE+Lu61TtHtTLz2RL9St744oMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDKzxLGqcjZRl9fPorhoMtE1fy9SIV5OouOd4IjBdxCzinMTOj5zazRCnxw9u1ZQHyjF1FMOvcwLtdpMti4DtQITQX/da82A0Wwf2/moqoQEZceDKMYxwZQFiIlwORqBADwmFjZkC9KCocFBrvS2FLEbaHvd9jheQZJUFj4X7cYGF4cricdmLF5kJ0N+DjKlToBm8FWOkPcgVG7doV52dpUDDJf+5dCn+GyW5uGxhb4PZRtVEEU4Z6V9CbR1GEfE42xwR/hL6qhK0SEtMPr0g5b3UxuDhMl2Yjewxq30VUdK84F9BfyGc7zkUSRhZguNfARgB", + "thinking": "The calculator tool returned an error: \"division by zero\". The user asked me not to recover from the error, so I'll just present the error as-is.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "Division by zero error.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 215, + "prompt_cache_creation_tokens": 205, + "prompt_cached_tokens": 37744, + "prompt_tokens": 37957, + "tokens": 38172 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json deleted file mode 100644 index 5d13ceeaa..000000000 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.79.span-events.json +++ /dev/null @@ -1,316 +0,0 @@ -{ - "async_prompt": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "async-prompt" - }, - "metric_keys": [], - "name": "claude-agent-async-prompt-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "basic": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to multiply 15 by 7. Do not answer from memory." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "basic" - }, - "metric_keys": [], - "name": "claude-agent-basic-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "failure": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to divide 2 by 0. Do not recover from the error." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "failure" - }, - "metric_keys": [], - "name": "claude-agent-failure-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "error": "division by zero", - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "claude-agent-sdk-traces" - }, - "metric_keys": [], - "name": "claude-agent-sdk-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "subagent": { - "handoff_tool": { - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "Agent" - }, - "metric_keys": [], - "name": "tool: Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "nested_task": { - "has_input": false, - "has_output": false, - "metadata": { - "claude_agent_sdk.agent_type": "math-expert", - "claude_agent_sdk.description": "", - "claude_agent_sdk.task_id": "", - "claude_agent_sdk.task_type": "local_agent", - "claude_agent_sdk.tool_use_id": "" - }, - "metric_keys": [], - "name": "Agent: ", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "subagent" - }, - "metric_keys": [], - "name": "claude-agent-subagent-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task_root": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "type": "tool" - } - } -} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.json new file mode 100644 index 000000000..5731f1d7f --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_0117rPYttev4aUWbEHmd8MNP", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuQDCmMIDRgCKkCgV2LfmZAW1ymvQL3NdfJrEYK4CtOclNE+4p4JpsgmafOw9/ntk4re8rVIaUwNHD3/8/rgvaKUwlg8s9S1Hh4DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCQFAZJvzlblaO85XhoMQuGhKMV6+JZOfHrEIjBr5UhLlxRT5+lUR/J3ayWG2SyRLjnIEbUc14v8y6syk3VsFCs708FteXRePHOKzWoqrgL0fC93Y3TA9bPiVYGIQjNM+eJuSB18QQ+Efq0HJaqNNKh26qbqEn1+blqYy24JVS6v0KheSETAs5FFtHcUvhcDlEw5yLbXZA3FxUE9vdikAzzOLMzxoo01MtVwA4bNoh1nLDoxq3z3F4dlzB5hZIVrcziIgaA7jkVoZWik2ntmW8JjI1lpnIVFWS+mWPp/wp5tOgSmWuqXrISCSRenxMYNyic6hbBL0fTEUvk/pv+9lPVnMRF60sd2APcsMJ5olY7nuC8gDk1s5DeWkYW9wN1WDlJKvTRNGeY9zgsxYACul49i9zrlg+BXK+CToOtt7i0HVQ9NjIA2Gn2VW5YIe8ePJCYVP8dqK0EnuGCNUJ15w5l8L+Yjy2ypd2kX9STOLPu9RCZjAQ74ka4MOzdtxxgB", + "thinking": "The user is asking me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory\", which means I should use the mcp__calculator__calculator tool to perform this calculation.\n\nLet me call the calculator tool with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 18917, + "prompt_tokens": 18927, + "tokens": 18934 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoMCCmMIDRgCKkB3qcw0i7BOpxsR3f7c6/lCjzjhBodFlnZ9rMgIBxxw0zIYSNT0h9n+7Dq1eNiiAJuUg4On5bSkq4Um2XSghCupMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDMBdGRjSqAtPercZ0hoMquUOgEP7LOsaN/vRIjAaYdHyn3DWUtvAcdWfBqKpBHt7uLTzDIu18PEveC2iHOYL/IJ4oEJ/KyBymsMe3RkqTlam4wGzQe5K3r7dL+R52kpNkgsIaMs11coQ1c9T3ahEtmAqj17uV0q5hjAGDEZuJDYl/9BBito759Dpbq1ivApDaZUeYsB15ZZ6sHf2/RgB", + "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "15 × 7 = **105**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 207, + "prompt_cache_creation_tokens": 204, + "prompt_cached_tokens": 37834, + "prompt_tokens": 38046, + "tokens": 38253 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "15 × 7 = **105**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuICCmMIDRgCKkAqhn3F2zkr6GBnjg4VTaP9xqmzrOPugMW/efy3E3yJB7XfwWM9uplMySB/GikQ4k2zvJWcFSwhPDhBOReMw1r8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCI16rquOdS7rbXnJBoMOzxOKnergSn7Rv4OIjDThwFMZOITmLoer89D2VK+Y+SKchUFKeBvL73uwn9kjrAlqQyj8GDOjtfseBltyOMqrAEKQWAkpPXvGwcTAoQj8IDT8rsZufbAvSP53zoqA2FGoGNF3zbDIUyUuSDRJItXNc/6wtziswtFqi5/6MOzbpNWO1PbXEFaB+oMxudE/R8k8Lr/EWylwAGCD612FMbKxSSpChRE7XCIosWv6aJgrx+QLPBa+ubbeEEfuVIe/4A+IufQOqBEq2UyTC0yWnBz3tS3S/FZ6i7aWzQkEbPe1yvWscFyn9WEHlLLmgIwGAE=", + "thinking": "The user has sent \"Part 1\" which is not a complete request. They haven't asked me to do anything specific yet. I should ask them what they'd like help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 171, + "prompt_cached_tokens": 19157, + "prompt_tokens": 19167, + "tokens": 19338 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EvkECmMIDRgCKkDSlcJB1WgMwafKBiCU1VfiGinXOLM6gcNh2Zd5UPST3CapZdAm6lWbFwI7PR4s9iocSm76gRk2U4vvc05Iam4iMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFGyuKpTdxNyeyUWWBoMQz+VuaxTYRQYtJKoIjCD9x009ku7cyujM6fgkDQzpgfur+batxicmCT/uZzsGvc+7KpAeeDwALNiQ2wjw+EqwwOdqSZO3VlNbPCmRrWalH8yox8COb7rHA8KADLeqp9VS21rbNrpQloo/oP3bglqWWh38oZ7/DsimzLxYKpVCUpt3uFF0N/WcA0OYvhEdTfnzaIkNWogUETF207+SQQ8luy4b7rkumqeqjmSOYLcucFXWP76C03VoN8a76beKEVmp8VGy7ZV84ZeYg8jfs4AVUDkafXc3boJHfrzej+MopYdmG5BZLXyTocfUeZBqhsuoYd4Y7cnSbllLeb6n+oBsp0OIOr8jnt+wJjIrmoe8dCU4ct9O51ikH5Zu/NXTqpT0r6+ihCrkGuZaZqbXCULR+K24Cmm72Lus18lIueAccfFYEhidcVQFIv8R9gP//hoi+SvoJDY+u8+aiCHXD2XOBc87YUyBja09s2U4rhiSRg7GWetkoPNlx5Me7DzCXtnLMIubRDTjk0w0AsNVaFK6YpjM78eBx8+ZrQHzXAsExRWIHHsyrUFltN1H6H0kNBXcdovpeX780zDjsiRdA66gbZJt+BkcdfAnNs0+JY5Quw5BL5C2QAFTFYRUYmcJB4gU/4VKAK0/uwa1EJRB2uMwAmPTM+9W2j+IIbZRw9IsNEhF+gJGAE=", + "thinking": "The user has now sent \"Part 2\" without any context. They seem to be sending fragments without explaining what they want. I should ask them to provide the complete request or context so I can understand what they're trying to accomplish.\n\nIt seems like they might be testing me, or they might have accidentally sent incomplete messages. Either way, I should politely ask for clarification about what task or project they'd like help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 49, + "prompt_cache_creation_tokens": 184, + "prompt_cached_tokens": 19157, + "prompt_tokens": 19351, + "tokens": 19400 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EsUDCmMIDRgCKkCpadVOiack3qbfD3/f7TAUdy8CR1tmjnyIwf7nrfCJCK2gcflkhrbRilTYvH+47ofKtoWYwldOkUAGnGUqIocaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPOLjjS7MSkNM+FTGRoMuCd4HIMMQN0wS800IjAiht4wCuKHGfp0XIfCK0MKai0s2s2f/b6cSruqsneK9G/l04IAeryKiLrN5/Z2tOkqjwIvo8rIK+tfYaEDkVAVIyCo5WvlZbsU80MaaREEjBwF77y1ij7Z1vBjoNDBJvWtBkKHXBxV4AfTR+MytuGrywJyqq+5sHzy4CDNgkaM803/CfDZKx0oaBZd8Egbu5RataQUbNmJI6U9T/MXDEqyAsC5UepuEcUNiBiaODEP+dVGjGV4ohdlu8CUoX7UlFUS5xcSkATqyJr2jMk+bABi3MKf1F+LiWvOS2ETxzz6OF32wMLPxLHgk2xRmC70xAA9f8KZ6Rs331Cl0EixKnzOs6udIXB9hv3cVJgVA9ydQ//3VK3hTy41XNe9OQ5S1P0X3Fmn95aaofobN25WITwGkkopPqWi6GppTuGy6N4qPY2rGAE=", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. I should use the Agent tool with subagent_type set to \"math-expert\" and ask it to add 15 and 27.\n\nLet me create the appropriate prompt for the math-expert agent.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 1, + "prompt_cached_tokens": 19318, + "prompt_tokens": 19328, + "tokens": 19329 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 together", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_014XWmbx353UYpHZSZN82BU3", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cache_creation_tokens": 9514, + "prompt_cached_tokens": 13239, + "prompt_tokens": 22756, + "tokens": 22764 + } + } + ], + "input": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "output": [ + { + "text": "The result is 42. (15 + 27 = 42)", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 together", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + "claude_agent_sdk.total_tokens": 22772 + } + } + ], + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The result is 42. (15 + 27 = 42)", + "type": "text" + } + ], + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22906, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 127 + }, + "cache_creation_input_tokens": 127, + "cache_read_input_tokens": 22753, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 20, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtQCCmMIDRgCKkASrw/kOUsOJ9+lO04xzRcguHYCw8i89WyVqakHDanNnLEvGYNfkduGkxcbaUjQ/kljvn7aIOfSBaR2dDoc0lf3MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN+GF8/AgJy5a3wkuxoM0x54pLnegBkIpHMGIjDFrroZD248rATm7/3KREHMlHZBwZxHASHdWFp2K/2E8+LN+BFdvuB1hviAjtSbrtIqngEQixeGsxK36fQVls3CquDgE2hhToqWKhJUT2mlW+tYDOMzoMLuQLuSpLg/RpA4Jz34KZv9RaJZRCTjbknVMtekwlUNFJD5HsncrihkEl+XoMtllHOfv5J9vQj52Zfku77xNfET+CRCgZo8YWyFtD/7MSnoX/528KO7/MmD5z7j8OQlDbCfsmnsYLiz9l6RFbsQ7bsQ9UFhwlvwb7a6NRgB", + "thinking": "Perfect! The math-expert subagent used the calculator tool to add 15 and 27 and got the result of 42. I should report this result to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 269, + "prompt_cache_creation_tokens": 298, + "prompt_cached_tokens": 38636, + "prompt_tokens": 38942, + "tokens": 39211 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtUGCmQIDRgCKkDxtVR9TJDmBhtkx+zbglgOXBnDvf7HmigKPWmisiAhyeGIMzRrSZVN4Oi0HqNAsZwgCKmHW90EWX5ou0OjIYrhMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy29yPv59LCEdPDLjoaDMM/bLL7dhTg3p0FKiIw9E3IYQG8JxM/zDiu0sjpzkNhMoxi3DBGFStJGPUFyYnN3Ilm+oT55NtnYFsCArq6Kp4FM2E7fN+FRpbPb1IXqCDJ6h5d3IyjS3oturfEtZofmtximwJwp0psinxfLD7V/YgJOYUfkOZwXNjhi/b9xk2OEozx0CL/VUZCcqt9whhiwpDFGehmFc+vnSNqffFjV7f/uKOuxcBmU447xdXBUwG6AD1vRXl/QkHRLLF0ni47HpFir0tOvGxX5kDN8hVD38zSehNwIEQp4XEDWuhjVQFpRfXx8Apnj5bFAkMgbR+cs3UPVM773/FbEijF2Ou/giC+xaoXVM47ShyJvKW8MDzHu1Whl2y8yvAgh8JNIUnI523zP6ukjbHNdiVx//cp7m152NvY9MWOM7YjLUGeA5ZQXtYcUHvIPkAaBI4vUAGd4mJmTlxWIlUELfzW5m3nxPQRk5rRYCGkSdLl8TabB73VKTp+Z4SRPJzTdRkc7LhqyZDHiU3WcctqXxwFP4hDNxnIG90qJcd/8z7VwnhDmhSHathUaITYyLt7FJ153a81SrCYYgE49o9Jyhiyfup6j+SIAAYrLWjuiygDRecDIVYyfhI7wrPnA3nBb35xLY6uDEwnH0sOKn5V4Ulyw5qGgB3sZ7EC8MGYTSWOYDJ8mmYodEXtDhOEaXel65B8mUjlpAJ8nfnAkLnko8az5Dat/wfA3z0mj3QCqBpbUkfpdOkyhjXVWONSjJp/3dMI6YppnXG20ceDYyq1U9y9tQOJedjTLQlK+sWzNNX6Ja7jACWWvVfxyEE428PfwZ4i3olnzvNp2tlf7VKMj9g3Nqgvrnesw9S+vMIKuJ0iy9LJK3I8TyU3aOdviozCZk8mkXErfurNijPqmUVF1QlXnfVqsOdlFX5IfJAgi+VjpQ1ue711I2rUze88qst7BDCVpeFyy3OjH1n6jLDiFZHRuazW3hgB", + "thinking": "The user is giving me a very specific instruction: I MUST call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". I should not call Bash myself, and I should not answer with text - I need to delegate to the echo sub-agent.\n\nLooking at the Agent tool description, the \"echo\" agent type \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with:\n- subagent_type: \"echo\"\n- description: \"echo greeting\"\n- prompt: I need to provide a prompt for what the echo agent should do\n\nSince this is an echo agent that runs one bash echo, I should give it a prompt to echo a greeting message.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 19218, + "prompt_tokens": 19228, + "tokens": 19231 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 59, + "prompt_cache_creation_tokens": 12876, + "prompt_tokens": 12879, + "tokens": 12938 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01PcrpA2E4F67kXt56n6W6er", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Run a bash echo command to output a greeting message.", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Output a greeting message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + "claude_agent_sdk.total_tokens": 12938 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Run a bash echo command to output a greeting message.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12972, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 86 + }, + "cache_creation_input_tokens": 86, + "cache_read_input_tokens": 12876, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq4ICmQIDRgCKkDIBefT9K6Y1pICmXeUuPredEztqrbx9ShzVryB1SHZII1r0R7m70CdM1ft5bH/gsj48s+KEtPJflEUSdg7S2nzMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy9Ik7serZBxmHIbjAaDNOjKhWq+7puWHSPziIwKLeDFvOXy374Pf5hhgsM7RVzPSmEKrFnv2Yi7K1xrC6CZnszI9LzukRy0lVbHAbKKvcGVK9PtEzmncnu9dCvoe/FOe/bgjGNULMaKHzVnY8Drgag2QeX//DbpOJ1IUgeILQvMSLVCP8DtaTUit0nxvoFZim4l73LEnA6SEWpZ/x4LnvfXVsA93nDBbBpHkbS+bjTlpWbLfmldufXcB22aAXhb1Y9KQW43OHcOMVDLLh2r8042zlrmBywEKofAen70hT3H6JvcRIK/hlwTDQaNpHiKqI8I2Gmy3R0L1rL79suIFK0eRpghFR9/B7+JeB8QmAFqn+Ihhby7hZ6kobGzwIh1OTOxsFUWdkEl1jsTD27yyqDE0sdiTHoD7Z8BTv1D5aDtfSo8HOTA/9G1M4QbnbawAe/Qf47DjzwhRnfXRvPi5dNK3YRJRpMWnrvez06owaKxcfleWSo6eLZDiKvmGYhfJlM/tjHO27PsQxB+vyIz0bR804XPOu1lGvMzMMEqHDUp3cU+WqR5OJaX0EJSkpjjgD/w+7ai5b3BcCiagc4YOPVZqqV/wsDSZc80uE2kqoj+KAv+tTygW+6xjYcX2HXbPc7xD49PCl72L1qc0fN/spTstDOo1xO4r3k9hjq7IWnIhCc1j0ct4GRAocYB2NzvVoixp7GRzdYMMUGYH4dw8GICE7DeTjFpVaOzAIJYzPDjXbCM0Yn/bRfehRAy5K0Tfn1Kz0e1LqONkaujXlFu7phzTPaxzB6woEg9vdL3V8GOlDggegJyUxJhBQWuOHnfII6RHmDdHRJKnRGNNrjystYg/3VR/xEGUnzxNjVNJeGcVU6f2cALKyyFnDnus/OQsOhsBuW6u0oomsP91g8SGJBVcyeaVLmiV6AM+fajVBmfedqUD00rNGLDMr9ICfkaTccILt3gG9ZjLVKWBIhBTqOUNYaU9pkinikD/bJnYBjIhUJ9drAl3Ftk1gyisPCguhN96RCz7pIIGVg2kE8RbcRlRsIZRCEu1bcDE73ZBL93COGtOUnldyG39PWbPC/jDh10mamHjfSccwB3UNT4TEdMXJQYFDOoDALXtS+6sWEzLO7JXjOxyXY0Ic/7w9Kc32Ui/82tjSMfmUWfj+mY/ecVSWuG2rcCN1DNXM/ppnE2CuRVp9MwZgzTtAKpKI6pnX/P+sEbixkipya4mjX6xpjrQfC3YiTGeYyPFcLZHwylS6JKGj1evVcb9/8QFNXKJ9tu73r56sYAQ==", + "thinking": "The echo agent has completed. According to the instructions, when an agent is done, it returns a single message back to me, and the result is not visible to the user. I should send a text message back to the user with a concise summary of the result.\n\nHowever, I notice the function result doesn't actually show me what the echo agent said or did - it just shows it completed with agent ID aa78b663afc5637a9.\n\nBut wait, looking more carefully at the instructions: \"When an agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nThe result here seems to just be \"done\" without any actual output. This is a bit unusual. Let me respond to the user that the echo agent has completed its task.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed and sent a greeting message.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 430, + "prompt_cache_creation_tokens": 362, + "prompt_cached_tokens": 38436, + "prompt_tokens": 38806, + "tokens": 39236 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed and sent a greeting message.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01HM53Dgu54gnzQEZMp744Ua", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "ErYECmMIDRgCKkBBXV54rxI9PiQXBXUh7+BZnW7Ej/LcxR6xa+6w53+YjsNNCH5pPtVLZs2SgMGo0dP1ekxe1LLBZ6VB7+3LNkItMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDNAhx6f1XtDIFDLdZhoMescY53N4oymlYKUwIjAuB+LUXhWcDodizE9nhNjhMi75aOPKOG74BdiAl45os3YoZW4K8Sm+p5GT1VkhtOIqgAMfS3JhW1A3ktD7tgpUF9hQi7jNPbpsqiyg9ScbYw10ATYORqJ/KJIeJyglFwuVBvNQlpDsUTHT07M8bKv5sP/MPR0hrFSQPF+6dSL5hyitpFfaUzGgb+lXegLNNMTTvhZEw1/9a/hNAITW8YuJE5RJeIaq56CJ/SwakrwAk8KxyEOVHYvKUiStaTAdYe8LGcH8Yy6ToPwocTmg8aV97291AqFTXBPyAAAgLw1Ix5uuzQ7hxZ2xnpZYUCD5uSBcbn92TFTFJSYXllHgpoy81DWbi/mSS6GQq3G0enVuvMXtMqHUfR40doHF7JW3VDGKVqR1kkAlggKDobJkc59wmeLeGEM0z/Goo10U2jus+CoPXYh/pHmmor7Lp8eDNfNJ2SEgmVNsBGyr8VT9vFg/GhOCQlpxMvZd5zRILdDwECh+t51bOVFG3yGbREjuLI5tci97yBLuamezYkeA1bIXZYgWp7zBlDzhRXDhSwZ3bPoHx71j5tsZIz3g7XVwufHEH6sYAQ==", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0, and explicitly tells me not to recover from the error. This is testing what happens when I attempt an invalid mathematical operation.\n\nLet me use the calculator tool with:\n- operation: \"divide\"\n- a: 2\n- b: 0\n\nThe user has explicitly asked me to attempt this and not recover from any error that occurs.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 19291, + "prompt_tokens": 19301, + "tokens": 19309 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq0DCmMIDRgCKkBycB6lZSvTw0Vn6nK+WvTd6RCisqfmwFHqfGlCUWt5cNemWVLFN/rO3NviniXxZhPj5trzQ8iHoafOdyoGDTT8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLHQaayWIYF0q6r7shoM9Ua7OzUkDWfKdNOIIjC4VCnTxXOXt4KbD7C4GbGjemoqargi2hPLax3rVbTeTXo1c0k+oNeyQUARCwe0dQgq9wHhDg3CdkVLQZHLyGcTgIDTuuYDDqa5QodSzTlc0KpAF8FKlzYYAD+j83xLSYRu9XmCHzntbUzZkvuqrIFyFTFNf2vTdgI+xrZo+zf9vXJSkmwk1f3PwFycVIkYk2yxokmqN71R+ZqTBpqrAAfAsWYWl4d+la8XFZ9biMYAYsJhHeZL7G763G9kw/ClUGvNxuQ23f8eBV6IVvJ0Ee8K+o9BmzLJb8jaIXfQyyLmLewrw9fS1vutUwcXrLF41Xoj926Log3MHPJjhyLRSPWgxaRJ1pIE8VvuxZVgAuo3WCxuZC9SyvkrcSXOmS9YlmxLPbFR74JVo0AyGAE=", + "thinking": "The calculator tool returned a division by zero error, as expected. The user specifically asked me not to recover from the error, so I should just present this error result without attempting to handle it or provide an alternative.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 246, + "prompt_cache_creation_tokens": 220, + "prompt_cached_tokens": 38582, + "prompt_tokens": 38810, + "tokens": 39056 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.txt new file mode 100644 index 000000000..b59ad2bed --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-auto-hook.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "15 × 7 = **105**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EuQDCmMIDRgCKkCgV2LfmZAW1ymvQL3NdfJrEYK4CtOclNE+4p4JpsgmafOw9/ntk4re8rVIaUwNHD3/8/rgvaKUwlg8s9S1Hh4DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCQFAZJvzlblaO85XhoMQuGhKMV6+JZOfHrEIjBr5UhLlxRT5+lUR/J3ayWG2SyRLjnIEbUc14v8y6syk3VsFCs708FteXRePHOKzWoqrgL0fC93Y3TA9bPiVYGIQjNM+eJuSB18QQ+Efq0HJaqNNKh26qbqEn1+blqYy24JVS6v0KheSETAs5FFtHcUvhcDlEw5yLbXZA3FxUE9vdikAzzOLMzxoo01MtVwA4bNoh1nLDoxq3z3F4dlzB5hZIVrcziIgaA7jkVoZWik2ntmW8JjI1lpnIVFWS+mWPp/wp5tOgSmWuqXrISCSRenxMYNyic6hbBL0fTEUvk/pv+9lPVnMRF60sd2APcsMJ5olY7nuC8gDk1s5DeWkYW9wN1WDlJKvTRNGeY9zgsxYACul49i9zrlg+BXK+CToOtt7i0HVQ9NjIA2Gn2VW5YIe8ePJCYVP8dqK0EnuGCNUJ15w5l8L+Yjy2ypd2kX9STOLPu9RCZjAQ74ka4MOzdtxxgB", + │ │ "thinking": "The user is asking me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory\", which means I should use the mcp__calculator__calculator tool to perform this calculation.\n\nLet me call the calculator tool with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cached_tokens": 18917, + │ │ "prompt_tokens": 18927, + │ │ "tokens": 18934 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_0117rPYttev4aUWbEHmd8MNP", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoMCCmMIDRgCKkB3qcw0i7BOpxsR3f7c6/lCjzjhBodFlnZ9rMgIBxxw0zIYSNT0h9n+7Dq1eNiiAJuUg4On5bSkq4Um2XSghCupMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDMBdGRjSqAtPercZ0hoMquUOgEP7LOsaN/vRIjAaYdHyn3DWUtvAcdWfBqKpBHt7uLTzDIu18PEveC2iHOYL/IJ4oEJ/KyBymsMe3RkqTlam4wGzQe5K3r7dL+R52kpNkgsIaMs11coQ1c9T3ahEtmAqj17uV0q5hjAGDEZuJDYl/9BBito759Dpbq1ivApDaZUeYsB15ZZ6sHf2/RgB", + │ "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "15 × 7 = **105**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 207, + │ "prompt_cache_creation_tokens": 204, + │ "prompt_cached_tokens": 37834, + │ "prompt_tokens": 38046, + │ "tokens": 38253 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EuICCmMIDRgCKkAqhn3F2zkr6GBnjg4VTaP9xqmzrOPugMW/efy3E3yJB7XfwWM9uplMySB/GikQ4k2zvJWcFSwhPDhBOReMw1r8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCI16rquOdS7rbXnJBoMOzxOKnergSn7Rv4OIjDThwFMZOITmLoer89D2VK+Y+SKchUFKeBvL73uwn9kjrAlqQyj8GDOjtfseBltyOMqrAEKQWAkpPXvGwcTAoQj8IDT8rsZufbAvSP53zoqA2FGoGNF3zbDIUyUuSDRJItXNc/6wtziswtFqi5/6MOzbpNWO1PbXEFaB+oMxudE/R8k8Lr/EWylwAGCD612FMbKxSSpChRE7XCIosWv6aJgrx+QLPBa+ubbeEEfuVIe/4A+IufQOqBEq2UyTC0yWnBz3tS3S/FZ6i7aWzQkEbPe1yvWscFyn9WEHlLLmgIwGAE=", + │ │ "thinking": "The user has sent \"Part 1\" which is not a complete request. They haven't asked me to do anything specific yet. I should ask them what they'd like help with.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 171, + │ │ "prompt_cached_tokens": 19157, + │ │ "prompt_tokens": 19167, + │ │ "tokens": 19338 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EvkECmMIDRgCKkDSlcJB1WgMwafKBiCU1VfiGinXOLM6gcNh2Zd5UPST3CapZdAm6lWbFwI7PR4s9iocSm76gRk2U4vvc05Iam4iMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFGyuKpTdxNyeyUWWBoMQz+VuaxTYRQYtJKoIjCD9x009ku7cyujM6fgkDQzpgfur+batxicmCT/uZzsGvc+7KpAeeDwALNiQ2wjw+EqwwOdqSZO3VlNbPCmRrWalH8yox8COb7rHA8KADLeqp9VS21rbNrpQloo/oP3bglqWWh38oZ7/DsimzLxYKpVCUpt3uFF0N/WcA0OYvhEdTfnzaIkNWogUETF207+SQQ8luy4b7rkumqeqjmSOYLcucFXWP76C03VoN8a76beKEVmp8VGy7ZV84ZeYg8jfs4AVUDkafXc3boJHfrzej+MopYdmG5BZLXyTocfUeZBqhsuoYd4Y7cnSbllLeb6n+oBsp0OIOr8jnt+wJjIrmoe8dCU4ct9O51ikH5Zu/NXTqpT0r6+ihCrkGuZaZqbXCULR+K24Cmm72Lus18lIueAccfFYEhidcVQFIv8R9gP//hoi+SvoJDY+u8+aiCHXD2XOBc87YUyBja09s2U4rhiSRg7GWetkoPNlx5Me7DzCXtnLMIubRDTjk0w0AsNVaFK6YpjM78eBx8+ZrQHzXAsExRWIHHsyrUFltN1H6H0kNBXcdovpeX780zDjsiRdA66gbZJt+BkcdfAnNs0+JY5Quw5BL5C2QAFTFYRUYmcJB4gU/4VKAK0/uwa1EJRB2uMwAmPTM+9W2j+IIbZRw9IsNEhF+gJGAE=", + │ "thinking": "The user has now sent \"Part 2\" without any context. They seem to be sending fragments without explaining what they want. I should ask them to provide the complete request or context so I can understand what they're trying to accomplish.\n\nIt seems like they might be testing me, or they might have accidentally sent incomplete messages. Either way, I should politely ask for clarification about what task or project they'd like help with.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 49, + │ "prompt_cache_creation_tokens": 184, + │ "prompt_cached_tokens": 19157, + │ "prompt_tokens": 19351, + │ "tokens": 19400 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EsUDCmMIDRgCKkCpadVOiack3qbfD3/f7TAUdy8CR1tmjnyIwf7nrfCJCK2gcflkhrbRilTYvH+47ofKtoWYwldOkUAGnGUqIocaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPOLjjS7MSkNM+FTGRoMuCd4HIMMQN0wS800IjAiht4wCuKHGfp0XIfCK0MKai0s2s2f/b6cSruqsneK9G/l04IAeryKiLrN5/Z2tOkqjwIvo8rIK+tfYaEDkVAVIyCo5WvlZbsU80MaaREEjBwF77y1ij7Z1vBjoNDBJvWtBkKHXBxV4AfTR+MytuGrywJyqq+5sHzy4CDNgkaM803/CfDZKx0oaBZd8Egbu5RataQUbNmJI6U9T/MXDEqyAsC5UepuEcUNiBiaODEP+dVGjGV4ohdlu8CUoX7UlFUS5xcSkATqyJr2jMk+bABi3MKf1F+LiWvOS2ETxzz6OF32wMLPxLHgk2xRmC70xAA9f8KZ6Rs331Cl0EixKnzOs6udIXB9hv3cVJgVA9ydQ//3VK3hTy41XNe9OQ5S1P0X3Fmn95aaofobN25WITwGkkopPqWi6GppTuGy6N4qPY2rGAE=", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. I should use the Agent tool with subagent_type set to \"math-expert\" and ask it to add 15 and 27.\n\nLet me create the appropriate prompt for the math-expert agent.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_cached_tokens": 19318, + │ │ "prompt_tokens": 19328, + │ │ "tokens": 19329 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The result is 42. (15 + 27 = 42)", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22906, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 127 + │ │ }, + │ │ "cache_creation_input_tokens": 127, + │ │ "cache_read_input_tokens": 22753, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 20, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 together [task] + │ │ input: "Please use the calculator tool to add 15 and 27 together. Report the result." + │ │ output: [ + │ │ { + │ │ "text": "The result is 42. (15 + 27 = 42)", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 together", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + │ │ "claude_agent_sdk.total_tokens": 22772 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cache_creation_tokens": 9514, + │ │ "prompt_cached_tokens": 13239, + │ │ "prompt_tokens": 22756, + │ │ "tokens": 22764 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_014XWmbx353UYpHZSZN82BU3", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 together", + │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EtQCCmMIDRgCKkASrw/kOUsOJ9+lO04xzRcguHYCw8i89WyVqakHDanNnLEvGYNfkduGkxcbaUjQ/kljvn7aIOfSBaR2dDoc0lf3MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN+GF8/AgJy5a3wkuxoM0x54pLnegBkIpHMGIjDFrroZD248rATm7/3KREHMlHZBwZxHASHdWFp2K/2E8+LN+BFdvuB1hviAjtSbrtIqngEQixeGsxK36fQVls3CquDgE2hhToqWKhJUT2mlW+tYDOMzoMLuQLuSpLg/RpA4Jz34KZv9RaJZRCTjbknVMtekwlUNFJD5HsncrihkEl+XoMtllHOfv5J9vQj52Zfku77xNfET+CRCgZo8YWyFtD/7MSnoX/528KO7/MmD5z7j8OQlDbCfsmnsYLiz9l6RFbsQ7bsQ9UFhwlvwb7a6NRgB", + │ "thinking": "Perfect! The math-expert subagent used the calculator tool to add 15 and 27 and got the result of 42. I should report this result to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 269, + │ "prompt_cache_creation_tokens": 298, + │ "prompt_cached_tokens": 38636, + │ "prompt_tokens": 38942, + │ "tokens": 39211 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed and sent a greeting message.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EtUGCmQIDRgCKkDxtVR9TJDmBhtkx+zbglgOXBnDvf7HmigKPWmisiAhyeGIMzRrSZVN4Oi0HqNAsZwgCKmHW90EWX5ou0OjIYrhMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy29yPv59LCEdPDLjoaDMM/bLL7dhTg3p0FKiIw9E3IYQG8JxM/zDiu0sjpzkNhMoxi3DBGFStJGPUFyYnN3Ilm+oT55NtnYFsCArq6Kp4FM2E7fN+FRpbPb1IXqCDJ6h5d3IyjS3oturfEtZofmtximwJwp0psinxfLD7V/YgJOYUfkOZwXNjhi/b9xk2OEozx0CL/VUZCcqt9whhiwpDFGehmFc+vnSNqffFjV7f/uKOuxcBmU447xdXBUwG6AD1vRXl/QkHRLLF0ni47HpFir0tOvGxX5kDN8hVD38zSehNwIEQp4XEDWuhjVQFpRfXx8Apnj5bFAkMgbR+cs3UPVM773/FbEijF2Ou/giC+xaoXVM47ShyJvKW8MDzHu1Whl2y8yvAgh8JNIUnI523zP6ukjbHNdiVx//cp7m152NvY9MWOM7YjLUGeA5ZQXtYcUHvIPkAaBI4vUAGd4mJmTlxWIlUELfzW5m3nxPQRk5rRYCGkSdLl8TabB73VKTp+Z4SRPJzTdRkc7LhqyZDHiU3WcctqXxwFP4hDNxnIG90qJcd/8z7VwnhDmhSHathUaITYyLt7FJ153a81SrCYYgE49o9Jyhiyfup6j+SIAAYrLWjuiygDRecDIVYyfhI7wrPnA3nBb35xLY6uDEwnH0sOKn5V4Ulyw5qGgB3sZ7EC8MGYTSWOYDJ8mmYodEXtDhOEaXel65B8mUjlpAJ8nfnAkLnko8az5Dat/wfA3z0mj3QCqBpbUkfpdOkyhjXVWONSjJp/3dMI6YppnXG20ceDYyq1U9y9tQOJedjTLQlK+sWzNNX6Ja7jACWWvVfxyEE428PfwZ4i3olnzvNp2tlf7VKMj9g3Nqgvrnesw9S+vMIKuJ0iy9LJK3I8TyU3aOdviozCZk8mkXErfurNijPqmUVF1QlXnfVqsOdlFX5IfJAgi+VjpQ1ue711I2rUze88qst7BDCVpeFyy3OjH1n6jLDiFZHRuazW3hgB", + │ │ "thinking": "The user is giving me a very specific instruction: I MUST call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". I should not call Bash myself, and I should not answer with text - I need to delegate to the echo sub-agent.\n\nLooking at the Agent tool description, the \"echo\" agent type \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with:\n- subagent_type: \"echo\"\n- description: \"echo greeting\"\n- prompt: I need to provide a prompt for what the echo agent should do\n\nSince this is an echo agent that runs one bash echo, I should give it a prompt to echo a greeting message.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 19218, + │ │ "prompt_tokens": 19228, + │ │ "tokens": 19231 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12972, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 86 + │ │ }, + │ │ "cache_creation_input_tokens": 86, + │ │ "cache_read_input_tokens": 12876, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Run a bash echo command to output a greeting message." + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Output a greeting message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + │ │ "claude_agent_sdk.total_tokens": 12938 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo \"hello\"", + │ │ │ "description": "Output a greeting message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 59, + │ │ │ "prompt_cache_creation_tokens": 12876, + │ │ │ "prompt_tokens": 12879, + │ │ │ "tokens": 12938 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo \"hello\"", + │ │ "description": "Output a greeting message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01PcrpA2E4F67kXt56n6W6er", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Run a bash echo command to output a greeting message.", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo \"hello\"", + │ "description": "Output a greeting message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Eq4ICmQIDRgCKkDIBefT9K6Y1pICmXeUuPredEztqrbx9ShzVryB1SHZII1r0R7m70CdM1ft5bH/gsj48s+KEtPJflEUSdg7S2nzMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy9Ik7serZBxmHIbjAaDNOjKhWq+7puWHSPziIwKLeDFvOXy374Pf5hhgsM7RVzPSmEKrFnv2Yi7K1xrC6CZnszI9LzukRy0lVbHAbKKvcGVK9PtEzmncnu9dCvoe/FOe/bgjGNULMaKHzVnY8Drgag2QeX//DbpOJ1IUgeILQvMSLVCP8DtaTUit0nxvoFZim4l73LEnA6SEWpZ/x4LnvfXVsA93nDBbBpHkbS+bjTlpWbLfmldufXcB22aAXhb1Y9KQW43OHcOMVDLLh2r8042zlrmBywEKofAen70hT3H6JvcRIK/hlwTDQaNpHiKqI8I2Gmy3R0L1rL79suIFK0eRpghFR9/B7+JeB8QmAFqn+Ihhby7hZ6kobGzwIh1OTOxsFUWdkEl1jsTD27yyqDE0sdiTHoD7Z8BTv1D5aDtfSo8HOTA/9G1M4QbnbawAe/Qf47DjzwhRnfXRvPi5dNK3YRJRpMWnrvez06owaKxcfleWSo6eLZDiKvmGYhfJlM/tjHO27PsQxB+vyIz0bR804XPOu1lGvMzMMEqHDUp3cU+WqR5OJaX0EJSkpjjgD/w+7ai5b3BcCiagc4YOPVZqqV/wsDSZc80uE2kqoj+KAv+tTygW+6xjYcX2HXbPc7xD49PCl72L1qc0fN/spTstDOo1xO4r3k9hjq7IWnIhCc1j0ct4GRAocYB2NzvVoixp7GRzdYMMUGYH4dw8GICE7DeTjFpVaOzAIJYzPDjXbCM0Yn/bRfehRAy5K0Tfn1Kz0e1LqONkaujXlFu7phzTPaxzB6woEg9vdL3V8GOlDggegJyUxJhBQWuOHnfII6RHmDdHRJKnRGNNrjystYg/3VR/xEGUnzxNjVNJeGcVU6f2cALKyyFnDnus/OQsOhsBuW6u0oomsP91g8SGJBVcyeaVLmiV6AM+fajVBmfedqUD00rNGLDMr9ICfkaTccILt3gG9ZjLVKWBIhBTqOUNYaU9pkinikD/bJnYBjIhUJ9drAl3Ftk1gyisPCguhN96RCz7pIIGVg2kE8RbcRlRsIZRCEu1bcDE73ZBL93COGtOUnldyG39PWbPC/jDh10mamHjfSccwB3UNT4TEdMXJQYFDOoDALXtS+6sWEzLO7JXjOxyXY0Ic/7w9Kc32Ui/82tjSMfmUWfj+mY/ecVSWuG2rcCN1DNXM/ppnE2CuRVp9MwZgzTtAKpKI6pnX/P+sEbixkipya4mjX6xpjrQfC3YiTGeYyPFcLZHwylS6JKGj1evVcb9/8QFNXKJ9tu73r56sYAQ==", + │ "thinking": "The echo agent has completed. According to the instructions, when an agent is done, it returns a single message back to me, and the result is not visible to the user. I should send a text message back to the user with a concise summary of the result.\n\nHowever, I notice the function result doesn't actually show me what the echo agent said or did - it just shows it completed with agent ID aa78b663afc5637a9.\n\nBut wait, looking more carefully at the instructions: \"When an agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nThe result here seems to just be \"done\" without any actual output. This is a bit unusual. Let me respond to the user that the echo agent has completed its task.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed and sent a greeting message.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 430, + │ "prompt_cache_creation_tokens": 362, + │ "prompt_cached_tokens": 38436, + │ "prompt_tokens": 38806, + │ "tokens": 39236 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "ErYECmMIDRgCKkBBXV54rxI9PiQXBXUh7+BZnW7Ej/LcxR6xa+6w53+YjsNNCH5pPtVLZs2SgMGo0dP1ekxe1LLBZ6VB7+3LNkItMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDNAhx6f1XtDIFDLdZhoMescY53N4oymlYKUwIjAuB+LUXhWcDodizE9nhNjhMi75aOPKOG74BdiAl45os3YoZW4K8Sm+p5GT1VkhtOIqgAMfS3JhW1A3ktD7tgpUF9hQi7jNPbpsqiyg9ScbYw10ATYORqJ/KJIeJyglFwuVBvNQlpDsUTHT07M8bKv5sP/MPR0hrFSQPF+6dSL5hyitpFfaUzGgb+lXegLNNMTTvhZEw1/9a/hNAITW8YuJE5RJeIaq56CJ/SwakrwAk8KxyEOVHYvKUiStaTAdYe8LGcH8Yy6ToPwocTmg8aV97291AqFTXBPyAAAgLw1Ix5uuzQ7hxZ2xnpZYUCD5uSBcbn92TFTFJSYXllHgpoy81DWbi/mSS6GQq3G0enVuvMXtMqHUfR40doHF7JW3VDGKVqR1kkAlggKDobJkc59wmeLeGEM0z/Goo10U2jus+CoPXYh/pHmmor7Lp8eDNfNJ2SEgmVNsBGyr8VT9vFg/GhOCQlpxMvZd5zRILdDwECh+t51bOVFG3yGbREjuLI5tci97yBLuamezYkeA1bIXZYgWp7zBlDzhRXDhSwZ3bPoHx71j5tsZIz3g7XVwufHEH6sYAQ==", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0, and explicitly tells me not to recover from the error. This is testing what happens when I attempt an invalid mathematical operation.\n\nLet me use the calculator tool with:\n- operation: \"divide\"\n- a: 2\n- b: 0\n\nThe user has explicitly asked me to attempt this and not recover from any error that occurs.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 19291, + │ "prompt_tokens": 19301, + │ "tokens": 19309 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01HM53Dgu54gnzQEZMp744Ua", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at traced.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at traced (/js/dist/index.mjs:0:0)\n at /e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0\n at runHandler (/js/dist/index.mjs:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "Eq0DCmMIDRgCKkBycB6lZSvTw0Vn6nK+WvTd6RCisqfmwFHqfGlCUWt5cNemWVLFN/rO3NviniXxZhPj5trzQ8iHoafOdyoGDTT8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLHQaayWIYF0q6r7shoM9Ua7OzUkDWfKdNOIIjC4VCnTxXOXt4KbD7C4GbGjemoqargi2hPLax3rVbTeTXo1c0k+oNeyQUARCwe0dQgq9wHhDg3CdkVLQZHLyGcTgIDTuuYDDqa5QodSzTlc0KpAF8FKlzYYAD+j83xLSYRu9XmCHzntbUzZkvuqrIFyFTFNf2vTdgI+xrZo+zf9vXJSkmwk1f3PwFycVIkYk2yxokmqN71R+ZqTBpqrAAfAsWYWl4d+la8XFZ9biMYAYsJhHeZL7G763G9kw/ClUGvNxuQ23f8eBV6IVvJ0Ee8K+o9BmzLJb8jaIXfQyyLmLewrw9fS1vutUwcXrLF41Xoj926Log3MHPJjhyLRSPWgxaRJ1pIE8VvuxZVgAuo3WCxuZC9SyvkrcSXOmS9YlmxLPbFR74JVo0AyGAE=", + "thinking": "The calculator tool returned a division by zero error, as expected. The user specifically asked me not to recover from the error, so I should just present this error result without attempting to handle it or provide an alternative.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 246, + "prompt_cache_creation_tokens": 220, + "prompt_cached_tokens": 38582, + "prompt_tokens": 38810, + "tokens": 39056 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.json new file mode 100644 index 000000000..ecb1df2ec --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.json @@ -0,0 +1,1140 @@ +{ + "span_tree": [ + { + "name": "claude-agent-sdk-root", + "type": "task", + "children": [ + { + "name": "claude-agent-basic-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-multiply", + "children": [] + } + ], + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "output": { + "content": [ + { + "text": "multiply(15, 7) = 105", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_0117rPYttev4aUWbEHmd8MNP", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuQDCmMIDRgCKkCgV2LfmZAW1ymvQL3NdfJrEYK4CtOclNE+4p4JpsgmafOw9/ntk4re8rVIaUwNHD3/8/rgvaKUwlg8s9S1Hh4DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCQFAZJvzlblaO85XhoMQuGhKMV6+JZOfHrEIjBr5UhLlxRT5+lUR/J3ayWG2SyRLjnIEbUc14v8y6syk3VsFCs708FteXRePHOKzWoqrgL0fC93Y3TA9bPiVYGIQjNM+eJuSB18QQ+Efq0HJaqNNKh26qbqEn1+blqYy24JVS6v0KheSETAs5FFtHcUvhcDlEw5yLbXZA3FxUE9vdikAzzOLMzxoo01MtVwA4bNoh1nLDoxq3z3F4dlzB5hZIVrcziIgaA7jkVoZWik2ntmW8JjI1lpnIVFWS+mWPp/wp5tOgSmWuqXrISCSRenxMYNyic6hbBL0fTEUvk/pv+9lPVnMRF60sd2APcsMJ5olY7nuC8gDk1s5DeWkYW9wN1WDlJKvTRNGeY9zgsxYACul49i9zrlg+BXK+CToOtt7i0HVQ9NjIA2Gn2VW5YIe8ePJCYVP8dqK0EnuGCNUJ15w5l8L+Yjy2ypd2kX9STOLPu9RCZjAQ74ka4MOzdtxxgB", + "thinking": "The user is asking me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory\", which means I should use the mcp__calculator__calculator tool to perform this calculation.\n\nLet me call the calculator tool with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 7, + "prompt_cached_tokens": 18917, + "prompt_tokens": 18927, + "tokens": 18934 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 7, + "operation": "multiply" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EoMCCmMIDRgCKkB3qcw0i7BOpxsR3f7c6/lCjzjhBodFlnZ9rMgIBxxw0zIYSNT0h9n+7Dq1eNiiAJuUg4On5bSkq4Um2XSghCupMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDMBdGRjSqAtPercZ0hoMquUOgEP7LOsaN/vRIjAaYdHyn3DWUtvAcdWfBqKpBHt7uLTzDIu18PEveC2iHOYL/IJ4oEJ/KyBymsMe3RkqTlam4wGzQe5K3r7dL+R52kpNkgsIaMs11coQ1c9T3ahEtmAqj17uV0q5hjAGDEZuJDYl/9BBito759Dpbq1ivApDaZUeYsB15ZZ6sHf2/RgB", + "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "15 × 7 = **105**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 207, + "prompt_cache_creation_tokens": 204, + "prompt_cached_tokens": 37834, + "prompt_tokens": 38046, + "tokens": 38253 + } + } + ], + "input": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + "output": { + "content": [ + { + "text": "15 × 7 = **105**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "claude-agent-async-prompt-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EuICCmMIDRgCKkAqhn3F2zkr6GBnjg4VTaP9xqmzrOPugMW/efy3E3yJB7XfwWM9uplMySB/GikQ4k2zvJWcFSwhPDhBOReMw1r8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCI16rquOdS7rbXnJBoMOzxOKnergSn7Rv4OIjDThwFMZOITmLoer89D2VK+Y+SKchUFKeBvL73uwn9kjrAlqQyj8GDOjtfseBltyOMqrAEKQWAkpPXvGwcTAoQj8IDT8rsZufbAvSP53zoqA2FGoGNF3zbDIUyUuSDRJItXNc/6wtziswtFqi5/6MOzbpNWO1PbXEFaB+oMxudE/R8k8Lr/EWylwAGCD612FMbKxSSpChRE7XCIosWv6aJgrx+QLPBa+ubbeEEfuVIe/4A+IufQOqBEq2UyTC0yWnBz3tS3S/FZ6i7aWzQkEbPe1yvWscFyn9WEHlLLmgIwGAE=", + "thinking": "The user has sent \"Part 1\" which is not a complete request. They haven't asked me to do anything specific yet. I should ask them what they'd like help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 171, + "prompt_cached_tokens": 19157, + "prompt_tokens": 19167, + "tokens": 19338 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Part 1", + "role": "user" + }, + { + "content": "Part 2", + "role": "user" + }, + { + "content": [ + { + "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + "type": "text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EvkECmMIDRgCKkDSlcJB1WgMwafKBiCU1VfiGinXOLM6gcNh2Zd5UPST3CapZdAm6lWbFwI7PR4s9iocSm76gRk2U4vvc05Iam4iMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFGyuKpTdxNyeyUWWBoMQz+VuaxTYRQYtJKoIjCD9x009ku7cyujM6fgkDQzpgfur+batxicmCT/uZzsGvc+7KpAeeDwALNiQ2wjw+EqwwOdqSZO3VlNbPCmRrWalH8yox8COb7rHA8KADLeqp9VS21rbNrpQloo/oP3bglqWWh38oZ7/DsimzLxYKpVCUpt3uFF0N/WcA0OYvhEdTfnzaIkNWogUETF207+SQQ8luy4b7rkumqeqjmSOYLcucFXWP76C03VoN8a76beKEVmp8VGy7ZV84ZeYg8jfs4AVUDkafXc3boJHfrzej+MopYdmG5BZLXyTocfUeZBqhsuoYd4Y7cnSbllLeb6n+oBsp0OIOr8jnt+wJjIrmoe8dCU4ct9O51ikH5Zu/NXTqpT0r6+ihCrkGuZaZqbXCULR+K24Cmm72Lus18lIueAccfFYEhidcVQFIv8R9gP//hoi+SvoJDY+u8+aiCHXD2XOBc87YUyBja09s2U4rhiSRg7GWetkoPNlx5Me7DzCXtnLMIubRDTjk0w0AsNVaFK6YpjM78eBx8+ZrQHzXAsExRWIHHsyrUFltN1H6H0kNBXcdovpeX780zDjsiRdA66gbZJt+BkcdfAnNs0+JY5Quw5BL5C2QAFTFYRUYmcJB4gU/4VKAK0/uwa1EJRB2uMwAmPTM+9W2j+IIbZRw9IsNEhF+gJGAE=", + "thinking": "The user has now sent \"Part 2\" without any context. They seem to be sending fragments without explaining what they want. I should ask them to provide the complete request or context so I can understand what they're trying to accomplish.\n\nIt seems like they might be testing me, or they might have accidentally sent incomplete messages. Either way, I should politely ask for clarification about what task or project they'd like help with.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 49, + "prompt_cache_creation_tokens": 184, + "prompt_cached_tokens": 19157, + "prompt_tokens": 19351, + "tokens": 19400 + } + } + ], + "input": [ + { + "message": { + "content": "Part 1", + "role": "user" + }, + "type": "user" + }, + { + "message": { + "content": "Part 2", + "role": "user" + }, + "type": "user" + } + ], + "output": { + "content": [ + { + "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "maxTurns": 1, + "model": "claude-haiku-4-5", + "num_turns": 1, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "async-prompt", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EsUDCmMIDRgCKkCpadVOiack3qbfD3/f7TAUdy8CR1tmjnyIwf7nrfCJCK2gcflkhrbRilTYvH+47ofKtoWYwldOkUAGnGUqIocaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPOLjjS7MSkNM+FTGRoMuCd4HIMMQN0wS800IjAiht4wCuKHGfp0XIfCK0MKai0s2s2f/b6cSruqsneK9G/l04IAeryKiLrN5/Z2tOkqjwIvo8rIK+tfYaEDkVAVIyCo5WvlZbsU80MaaREEjBwF77y1ij7Z1vBjoNDBJvWtBkKHXBxV4AfTR+MytuGrywJyqq+5sHzy4CDNgkaM803/CfDZKx0oaBZd8Egbu5RataQUbNmJI6U9T/MXDEqyAsC5UepuEcUNiBiaODEP+dVGjGV4ohdlu8CUoX7UlFUS5xcSkATqyJr2jMk+bABi3MKf1F+LiWvOS2ETxzz6OF32wMLPxLHgk2xRmC70xAA9f8KZ6Rs331Cl0EixKnzOs6udIXB9hv3cVJgVA9ydQ//3VK3hTy41XNe9OQ5S1P0X3Fmn95aaofobN25WITwGkkopPqWi6GppTuGy6N4qPY2rGAE=", + "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. I should use the Agent tool with subagent_type set to \"math-expert\" and ask it to add 15 and 27.\n\nLet me create the appropriate prompt for the math-expert agent.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 1, + "prompt_cached_tokens": 19318, + "prompt_tokens": 19328, + "tokens": 19329 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: Add 15 and 27 together", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-add", + "children": [] + } + ], + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "output": { + "content": [ + { + "text": "add(15, 27) = 42", + "type": "text" + } + ] + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_014XWmbx353UYpHZSZN82BU3", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + } + } + ], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cache_creation_tokens": 9514, + "prompt_cached_tokens": 13239, + "prompt_tokens": 22756, + "tokens": 22764 + } + } + ], + "input": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "output": [ + { + "text": "The result is 42. (15 + 27 = 42)", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "math-expert", + "claude_agent_sdk.description": "Add 15 and 27 together", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + "claude_agent_sdk.total_tokens": 22772 + } + } + ], + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "The result is 42. (15 + 27 = 42)", + "type": "text" + } + ], + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 22906, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 127 + }, + "cache_creation_input_tokens": 127, + "cache_read_input_tokens": 22753, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 20, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "Add 15 and 27 together", + "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + "subagent_type": "math-expert" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 15, + "b": 27, + "operation": "add" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtQCCmMIDRgCKkASrw/kOUsOJ9+lO04xzRcguHYCw8i89WyVqakHDanNnLEvGYNfkduGkxcbaUjQ/kljvn7aIOfSBaR2dDoc0lf3MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN+GF8/AgJy5a3wkuxoM0x54pLnegBkIpHMGIjDFrroZD248rATm7/3KREHMlHZBwZxHASHdWFp2K/2E8+LN+BFdvuB1hviAjtSbrtIqngEQixeGsxK36fQVls3CquDgE2hhToqWKhJUT2mlW+tYDOMzoMLuQLuSpLg/RpA4Jz34KZv9RaJZRCTjbknVMtekwlUNFJD5HsncrihkEl+XoMtllHOfv5J9vQj52Zfku77xNfET+CRCgZo8YWyFtD/7MSnoX/528KO7/MmD5z7j8OQlDbCfsmnsYLiz9l6RFbsQ7bsQ9UFhwlvwb7a6NRgB", + "thinking": "Perfect! The math-expert subagent used the calculator tool to add 15 and 27 and got the result of 42. I should report this result to the user.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 269, + "prompt_cache_creation_tokens": 298, + "prompt_cached_tokens": 38636, + "prompt_tokens": 38942, + "tokens": 39211 + } + } + ], + "input": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + "output": { + "content": [ + { + "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Task" + ], + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent", + "testRunId": "" + } + }, + { + "name": "claude-agent-subagent-built-in-tool-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "EtUGCmQIDRgCKkDxtVR9TJDmBhtkx+zbglgOXBnDvf7HmigKPWmisiAhyeGIMzRrSZVN4Oi0HqNAsZwgCKmHW90EWX5ou0OjIYrhMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy29yPv59LCEdPDLjoaDMM/bLL7dhTg3p0FKiIw9E3IYQG8JxM/zDiu0sjpzkNhMoxi3DBGFStJGPUFyYnN3Ilm+oT55NtnYFsCArq6Kp4FM2E7fN+FRpbPb1IXqCDJ6h5d3IyjS3oturfEtZofmtximwJwp0psinxfLD7V/YgJOYUfkOZwXNjhi/b9xk2OEozx0CL/VUZCcqt9whhiwpDFGehmFc+vnSNqffFjV7f/uKOuxcBmU447xdXBUwG6AD1vRXl/QkHRLLF0ni47HpFir0tOvGxX5kDN8hVD38zSehNwIEQp4XEDWuhjVQFpRfXx8Apnj5bFAkMgbR+cs3UPVM773/FbEijF2Ou/giC+xaoXVM47ShyJvKW8MDzHu1Whl2y8yvAgh8JNIUnI523zP6ukjbHNdiVx//cp7m152NvY9MWOM7YjLUGeA5ZQXtYcUHvIPkAaBI4vUAGd4mJmTlxWIlUELfzW5m3nxPQRk5rRYCGkSdLl8TabB73VKTp+Z4SRPJzTdRkc7LhqyZDHiU3WcctqXxwFP4hDNxnIG90qJcd/8z7VwnhDmhSHathUaITYyLt7FJ153a81SrCYYgE49o9Jyhiyfup6j+SIAAYrLWjuiygDRecDIVYyfhI7wrPnA3nBb35xLY6uDEwnH0sOKn5V4Ulyw5qGgB3sZ7EC8MGYTSWOYDJ8mmYodEXtDhOEaXel65B8mUjlpAJ8nfnAkLnko8az5Dat/wfA3z0mj3QCqBpbUkfpdOkyhjXVWONSjJp/3dMI6YppnXG20ceDYyq1U9y9tQOJedjTLQlK+sWzNNX6Ja7jACWWvVfxyEE428PfwZ4i3olnzvNp2tlf7VKMj9g3Nqgvrnesw9S+vMIKuJ0iy9LJK3I8TyU3aOdviozCZk8mkXErfurNijPqmUVF1QlXnfVqsOdlFX5IfJAgi+VjpQ1ue711I2rUze88qst7BDCVpeFyy3OjH1n6jLDiFZHRuazW3hgB", + "thinking": "The user is giving me a very specific instruction: I MUST call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". I should not call Bash myself, and I should not answer with text - I need to delegate to the echo sub-agent.\n\nLooking at the Agent tool description, the \"echo\" agent type \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with:\n- subagent_type: \"echo\"\n- description: \"echo greeting\"\n- prompt: I need to provide a prompt for what the echo agent should do\n\nSince this is an echo agent that runs one bash echo, I should give it a prompt to echo a greeting message.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 19218, + "prompt_tokens": 19228, + "tokens": 19231 + } + }, + { + "name": "tool: Agent", + "type": "tool", + "children": [ + { + "name": "Agent: echo greeting", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 59, + "prompt_cache_creation_tokens": 12876, + "prompt_tokens": 12879, + "tokens": 12938 + } + }, + { + "name": "tool: Bash", + "type": "tool", + "children": [], + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "output": { + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "stderr": "", + "stdout": "hello" + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Bash", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01PcrpA2E4F67kXt56n6W6er", + "gen_ai.tool.name": "Bash" + } + } + ], + "input": "Run a bash echo command to output a greeting message.", + "output": [ + { + "text": "done", + "type": "text" + } + ], + "metadata": { + "claude_agent_sdk.agent_type": "echo", + "claude_agent_sdk.description": "Running Output a greeting message", + "claude_agent_sdk.duration_ms": 0, + "claude_agent_sdk.last_tool_name": "Bash", + "claude_agent_sdk.status": "completed", + "claude_agent_sdk.task_id": "", + "claude_agent_sdk.task_type": "local_agent", + "claude_agent_sdk.tool_use_count": 1, + "claude_agent_sdk.tool_use_id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + "claude_agent_sdk.total_tokens": 12938 + } + } + ], + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "output": { + "agentId": "", + "content": [ + { + "text": "done", + "type": "text" + } + ], + "prompt": "Run a bash echo command to output a greeting message.", + "status": "completed", + "totalDurationMs": 0, + "totalTokens": 12972, + "totalToolUseCount": 1, + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 86 + }, + "cache_creation_input_tokens": 86, + "cache_read_input_tokens": 12876, + "inference_geo": "", + "input_tokens": 6, + "iterations": [], + "output_tokens": 4, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 0 + }, + "service_tier": "standard", + "speed": "standard" + } + }, + "metadata": { + "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + "claude_agent_sdk.raw_tool_name": "Agent", + "claude_agent_sdk.session_id": "", + "gen_ai.tool.call.id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + "gen_ai.tool.name": "Agent" + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "description": "echo greeting", + "prompt": "Run a bash echo command to output a greeting message.", + "subagent_type": "echo" + }, + "name": "Agent", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "command": "echo \"hello\"", + "description": "Output a greeting message" + }, + "name": "Bash", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq4ICmQIDRgCKkDIBefT9K6Y1pICmXeUuPredEztqrbx9ShzVryB1SHZII1r0R7m70CdM1ft5bH/gsj48s+KEtPJflEUSdg7S2nzMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy9Ik7serZBxmHIbjAaDNOjKhWq+7puWHSPziIwKLeDFvOXy374Pf5hhgsM7RVzPSmEKrFnv2Yi7K1xrC6CZnszI9LzukRy0lVbHAbKKvcGVK9PtEzmncnu9dCvoe/FOe/bgjGNULMaKHzVnY8Drgag2QeX//DbpOJ1IUgeILQvMSLVCP8DtaTUit0nxvoFZim4l73LEnA6SEWpZ/x4LnvfXVsA93nDBbBpHkbS+bjTlpWbLfmldufXcB22aAXhb1Y9KQW43OHcOMVDLLh2r8042zlrmBywEKofAen70hT3H6JvcRIK/hlwTDQaNpHiKqI8I2Gmy3R0L1rL79suIFK0eRpghFR9/B7+JeB8QmAFqn+Ihhby7hZ6kobGzwIh1OTOxsFUWdkEl1jsTD27yyqDE0sdiTHoD7Z8BTv1D5aDtfSo8HOTA/9G1M4QbnbawAe/Qf47DjzwhRnfXRvPi5dNK3YRJRpMWnrvez06owaKxcfleWSo6eLZDiKvmGYhfJlM/tjHO27PsQxB+vyIz0bR804XPOu1lGvMzMMEqHDUp3cU+WqR5OJaX0EJSkpjjgD/w+7ai5b3BcCiagc4YOPVZqqV/wsDSZc80uE2kqoj+KAv+tTygW+6xjYcX2HXbPc7xD49PCl72L1qc0fN/spTstDOo1xO4r3k9hjq7IWnIhCc1j0ct4GRAocYB2NzvVoixp7GRzdYMMUGYH4dw8GICE7DeTjFpVaOzAIJYzPDjXbCM0Yn/bRfehRAy5K0Tfn1Kz0e1LqONkaujXlFu7phzTPaxzB6woEg9vdL3V8GOlDggegJyUxJhBQWuOHnfII6RHmDdHRJKnRGNNrjystYg/3VR/xEGUnzxNjVNJeGcVU6f2cALKyyFnDnus/OQsOhsBuW6u0oomsP91g8SGJBVcyeaVLmiV6AM+fajVBmfedqUD00rNGLDMr9ICfkaTccILt3gG9ZjLVKWBIhBTqOUNYaU9pkinikD/bJnYBjIhUJ9drAl3Ftk1gyisPCguhN96RCz7pIIGVg2kE8RbcRlRsIZRCEu1bcDE73ZBL93COGtOUnldyG39PWbPC/jDh10mamHjfSccwB3UNT4TEdMXJQYFDOoDALXtS+6sWEzLO7JXjOxyXY0Ic/7w9Kc32Ui/82tjSMfmUWfj+mY/ecVSWuG2rcCN1DNXM/ppnE2CuRVp9MwZgzTtAKpKI6pnX/P+sEbixkipya4mjX6xpjrQfC3YiTGeYyPFcLZHwylS6JKGj1evVcb9/8QFNXKJ9tu73r56sYAQ==", + "thinking": "The echo agent has completed. According to the instructions, when an agent is done, it returns a single message back to me, and the result is not visible to the user. I should send a text message back to the user with a concise summary of the result.\n\nHowever, I notice the function result doesn't actually show me what the echo agent said or did - it just shows it completed with agent ID aa78b663afc5637a9.\n\nBut wait, looking more carefully at the instructions: \"When an agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nThe result here seems to just be \"done\" without any actual output. This is a bit unusual. Let me respond to the user that the echo agent has completed its task.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The echo agent has completed and sent a greeting message.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-sonnet-4-5-20250929" + }, + "metrics": { + "completion_tokens": 430, + "prompt_cache_creation_tokens": 362, + "prompt_cached_tokens": 38436, + "prompt_tokens": 38806, + "tokens": 39236 + } + } + ], + "input": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + "output": { + "content": [ + { + "text": "The echo agent has completed and sent a greeting message.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "allowedTools": [ + "Agent", + "Bash" + ], + "model": "claude-sonnet-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "subagent-built-in-tool", + "testRunId": "" + } + }, + { + "name": "claude-agent-failure-operation", + "children": [ + { + "name": "Claude Agent", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [ + { + "name": "tool: calculator/calculator", + "type": "tool", + "children": [ + { + "name": "calculator-local-handler-divide", + "children": [], + "error": "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + } + ], + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "metadata": { + "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + "gen_ai.tool.call.id": "toolu_01HM53Dgu54gnzQEZMp744Ua", + "gen_ai.tool.name": "calculator", + "mcp.server": "calculator" + }, + "error": "division by zero" + } + ], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + } + ], + "output": [ + { + "content": [ + { + "signature": "ErYECmMIDRgCKkBBXV54rxI9PiQXBXUh7+BZnW7Ej/LcxR6xa+6w53+YjsNNCH5pPtVLZs2SgMGo0dP1ekxe1LLBZ6VB7+3LNkItMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDNAhx6f1XtDIFDLdZhoMescY53N4oymlYKUwIjAuB+LUXhWcDodizE9nhNjhMi75aOPKOG74BdiAl45os3YoZW4K8Sm+p5GT1VkhtOIqgAMfS3JhW1A3ktD7tgpUF9hQi7jNPbpsqiyg9ScbYw10ATYORqJ/KJIeJyglFwuVBvNQlpDsUTHT07M8bKv5sP/MPR0hrFSQPF+6dSL5hyitpFfaUzGgb+lXegLNNMTTvhZEw1/9a/hNAITW8YuJE5RJeIaq56CJ/SwakrwAk8KxyEOVHYvKUiStaTAdYe8LGcH8Yy6ToPwocTmg8aV97291AqFTXBPyAAAgLw1Ix5uuzQ7hxZ2xnpZYUCD5uSBcbn92TFTFJSYXllHgpoy81DWbi/mSS6GQq3G0enVuvMXtMqHUfR40doHF7JW3VDGKVqR1kkAlggKDobJkc59wmeLeGEM0z/Goo10U2jus+CoPXYh/pHmmor7Lp8eDNfNJ2SEgmVNsBGyr8VT9vFg/GhOCQlpxMvZd5zRILdDwECh+t51bOVFG3yGbREjuLI5tci97yBLuamezYkeA1bIXZYgWp7zBlDzhRXDhSwZ3bPoHx71j5tsZIz3g7XVwufHEH6sYAQ==", + "thinking": "The user is asking me to use the calculator tool to divide 2 by 0, and explicitly tells me not to recover from the error. This is testing what happens when I attempt an invalid mathematical operation.\n\nLet me use the calculator tool with:\n- operation: \"divide\"\n- a: 2\n- b: 0\n\nThe user has explicitly asked me to attempt this and not recover from any error that occurs.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 19291, + "prompt_tokens": 19301, + "tokens": 19309 + } + }, + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "signature": "Eq0DCmMIDRgCKkBycB6lZSvTw0Vn6nK+WvTd6RCisqfmwFHqfGlCUWt5cNemWVLFN/rO3NviniXxZhPj5trzQ8iHoafOdyoGDTT8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLHQaayWIYF0q6r7shoM9Ua7OzUkDWfKdNOIIjC4VCnTxXOXt4KbD7C4GbGjemoqargi2hPLax3rVbTeTXo1c0k+oNeyQUARCwe0dQgq9wHhDg3CdkVLQZHLyGcTgIDTuuYDDqa5QodSzTlc0KpAF8FKlzYYAD+j83xLSYRu9XmCHzntbUzZkvuqrIFyFTFNf2vTdgI+xrZo+zf9vXJSkmwk1f3PwFycVIkYk2yxokmqN71R+ZqTBpqrAAfAsWYWl4d+la8XFZ9biMYAYsJhHeZL7G763G9kw/ClUGvNxuQ23f8eBV6IVvJ0Ee8K+o9BmzLJb8jaIXfQyyLmLewrw9fS1vutUwcXrLF41Xoj926Log3MHPJjhyLRSPWgxaRJ1pIE8VvuxZVgAuo3WCxuZC9SyvkrcSXOmS9YlmxLPbFR74JVo0AyGAE=", + "thinking": "The calculator tool returned a division by zero error, as expected. The user specifically asked me not to recover from the error, so I should just present this error result without attempting to handle it or provide an alternative.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + ], + "metadata": { + "model": "claude-haiku-4-5-20251001" + }, + "metrics": { + "completion_tokens": 246, + "prompt_cache_creation_tokens": 220, + "prompt_cached_tokens": 38582, + "prompt_tokens": 38810, + "tokens": 39056 + } + } + ], + "input": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "output": { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + } + ], + "metadata": { + "operation": "failure", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.txt b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.txt new file mode 100644 index 000000000..c795904c0 --- /dev/null +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81-wrapped.span-tree.txt @@ -0,0 +1,988 @@ +span_tree: +└── claude-agent-sdk-root [task] + metadata: { + "scenario": "claude-agent-sdk-traces", + "testRunId": "" + } + ├── claude-agent-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Use the calculator tool to multiply 15 by 7. Do not answer from memory." + │ output: { + │ "content": [ + │ { + │ "text": "15 × 7 = **105**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EuQDCmMIDRgCKkCgV2LfmZAW1ymvQL3NdfJrEYK4CtOclNE+4p4JpsgmafOw9/ntk4re8rVIaUwNHD3/8/rgvaKUwlg8s9S1Hh4DMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCQFAZJvzlblaO85XhoMQuGhKMV6+JZOfHrEIjBr5UhLlxRT5+lUR/J3ayWG2SyRLjnIEbUc14v8y6syk3VsFCs708FteXRePHOKzWoqrgL0fC93Y3TA9bPiVYGIQjNM+eJuSB18QQ+Efq0HJaqNNKh26qbqEn1+blqYy24JVS6v0KheSETAs5FFtHcUvhcDlEw5yLbXZA3FxUE9vdikAzzOLMzxoo01MtVwA4bNoh1nLDoxq3z3F4dlzB5hZIVrcziIgaA7jkVoZWik2ntmW8JjI1lpnIVFWS+mWPp/wp5tOgSmWuqXrISCSRenxMYNyic6hbBL0fTEUvk/pv+9lPVnMRF60sd2APcsMJ5olY7nuC8gDk1s5DeWkYW9wN1WDlJKvTRNGeY9zgsxYACul49i9zrlg+BXK+CToOtt7i0HVQ9NjIA2Gn2VW5YIe8ePJCYVP8dqK0EnuGCNUJ15w5l8L+Yjy2ypd2kX9STOLPu9RCZjAQ74ka4MOzdtxxgB", + │ │ "thinking": "The user is asking me to multiply 15 by 7 using the calculator tool. They explicitly say \"Do not answer from memory\", which means I should use the mcp__calculator__calculator tool to perform this calculation.\n\nLet me call the calculator tool with:\n- operation: \"multiply\"\n- a: 15\n- b: 7", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 7, + │ │ "prompt_cached_tokens": 18917, + │ │ "prompt_tokens": 18927, + │ │ "tokens": 18934 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 7, + │ │ "operation": "multiply" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "multiply(15, 7) = 105", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_0117rPYttev4aUWbEHmd8MNP", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-multiply + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to multiply 15 by 7. Do not answer from memory.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 7, + │ "operation": "multiply" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EoMCCmMIDRgCKkB3qcw0i7BOpxsR3f7c6/lCjzjhBodFlnZ9rMgIBxxw0zIYSNT0h9n+7Dq1eNiiAJuUg4On5bSkq4Um2XSghCupMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDMBdGRjSqAtPercZ0hoMquUOgEP7LOsaN/vRIjAaYdHyn3DWUtvAcdWfBqKpBHt7uLTzDIu18PEveC2iHOYL/IJ4oEJ/KyBymsMe3RkqTlam4wGzQe5K3r7dL+R52kpNkgsIaMs11coQ1c9T3ahEtmAqj17uV0q5hjAGDEZuJDYl/9BBito759Dpbq1ivApDaZUeYsB15ZZ6sHf2/RgB", + │ "thinking": "Great! The calculator tool returned the result: 15 × 7 = 105.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "15 × 7 = **105**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 207, + │ "prompt_cache_creation_tokens": 204, + │ "prompt_cached_tokens": 37834, + │ "prompt_tokens": 38046, + │ "tokens": 38253 + │ } + ├── claude-agent-async-prompt-operation + │ metadata: { + │ "operation": "async-prompt", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: [ + │ { + │ "message": { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ "type": "user" + │ }, + │ { + │ "message": { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ "type": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "maxTurns": 1, + │ "model": "claude-haiku-4-5", + │ "num_turns": 1, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Part 1", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": "Part 2", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EuICCmMIDRgCKkAqhn3F2zkr6GBnjg4VTaP9xqmzrOPugMW/efy3E3yJB7XfwWM9uplMySB/GikQ4k2zvJWcFSwhPDhBOReMw1r8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDCI16rquOdS7rbXnJBoMOzxOKnergSn7Rv4OIjDThwFMZOITmLoer89D2VK+Y+SKchUFKeBvL73uwn9kjrAlqQyj8GDOjtfseBltyOMqrAEKQWAkpPXvGwcTAoQj8IDT8rsZufbAvSP53zoqA2FGoGNF3zbDIUyUuSDRJItXNc/6wtziswtFqi5/6MOzbpNWO1PbXEFaB+oMxudE/R8k8Lr/EWylwAGCD612FMbKxSSpChRE7XCIosWv6aJgrx+QLPBa+ubbeEEfuVIe/4A+IufQOqBEq2UyTC0yWnBz3tS3S/FZ6i7aWzQkEbPe1yvWscFyn9WEHlLLmgIwGAE=", + │ │ "thinking": "The user has sent \"Part 1\" which is not a complete request. They haven't asked me to do anything specific yet. I should ask them what they'd like help with.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 171, + │ │ "prompt_cached_tokens": 19157, + │ │ "prompt_tokens": 19167, + │ │ "tokens": 19338 + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Part 1", + │ "role": "user" + │ }, + │ { + │ "content": "Part 2", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I'm ready to help! However, I notice you've sent \"Part 1\" without any context about what you'd like me to do. \n\nCould you please provide more details about what you need? For example:\n- Are you working on a coding task?\n- Do you need help with a specific project or feature?\n- Is there a file or codebase you'd like me to work with?\n- Do you have multiple parts to a task you'd like to tackle?\n\nPlease share what you'd like to work on, and I'll be happy to assist!", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EvkECmMIDRgCKkDSlcJB1WgMwafKBiCU1VfiGinXOLM6gcNh2Zd5UPST3CapZdAm6lWbFwI7PR4s9iocSm76gRk2U4vvc05Iam4iMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDFGyuKpTdxNyeyUWWBoMQz+VuaxTYRQYtJKoIjCD9x009ku7cyujM6fgkDQzpgfur+batxicmCT/uZzsGvc+7KpAeeDwALNiQ2wjw+EqwwOdqSZO3VlNbPCmRrWalH8yox8COb7rHA8KADLeqp9VS21rbNrpQloo/oP3bglqWWh38oZ7/DsimzLxYKpVCUpt3uFF0N/WcA0OYvhEdTfnzaIkNWogUETF207+SQQ8luy4b7rkumqeqjmSOYLcucFXWP76C03VoN8a76beKEVmp8VGy7ZV84ZeYg8jfs4AVUDkafXc3boJHfrzej+MopYdmG5BZLXyTocfUeZBqhsuoYd4Y7cnSbllLeb6n+oBsp0OIOr8jnt+wJjIrmoe8dCU4ct9O51ikH5Zu/NXTqpT0r6+ihCrkGuZaZqbXCULR+K24Cmm72Lus18lIueAccfFYEhidcVQFIv8R9gP//hoi+SvoJDY+u8+aiCHXD2XOBc87YUyBja09s2U4rhiSRg7GWetkoPNlx5Me7DzCXtnLMIubRDTjk0w0AsNVaFK6YpjM78eBx8+ZrQHzXAsExRWIHHsyrUFltN1H6H0kNBXcdovpeX780zDjsiRdA66gbZJt+BkcdfAnNs0+JY5Quw5BL5C2QAFTFYRUYmcJB4gU/4VKAK0/uwa1EJRB2uMwAmPTM+9W2j+IIbZRw9IsNEhF+gJGAE=", + │ "thinking": "The user has now sent \"Part 2\" without any context. They seem to be sending fragments without explaining what they want. I should ask them to provide the complete request or context so I can understand what they're trying to accomplish.\n\nIt seems like they might be testing me, or they might have accidentally sent incomplete messages. Either way, I should politely ask for clarification about what task or project they'd like help with.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "I notice you're sending numbered parts without context. I'm still not sure what you're asking me to do!\n\nCould you please share:\n1. **What is the full task or question?** (What do you need help with?)\n2. **What's the context?** (Are you working on code, a project, research, etc.?)\n3. **What are the parts for?** (Are these separate tasks, or pieces of one larger task?)\n\nOnce you provide the complete information, I'll be ready to help you right away! 😊", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 49, + │ "prompt_cache_creation_tokens": 184, + │ "prompt_cached_tokens": 19157, + │ "prompt_tokens": 19351, + │ "tokens": 19400 + │ } + ├── claude-agent-subagent-operation + │ metadata: { + │ "operation": "subagent", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." + │ output: { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Task" + │ ], + │ "model": "claude-haiku-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EsUDCmMIDRgCKkCpadVOiack3qbfD3/f7TAUdy8CR1tmjnyIwf7nrfCJCK2gcflkhrbRilTYvH+47ofKtoWYwldOkUAGnGUqIocaMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDPOLjjS7MSkNM+FTGRoMuCd4HIMMQN0wS800IjAiht4wCuKHGfp0XIfCK0MKai0s2s2f/b6cSruqsneK9G/l04IAeryKiLrN5/Z2tOkqjwIvo8rIK+tfYaEDkVAVIyCo5WvlZbsU80MaaREEjBwF77y1ij7Z1vBjoNDBJvWtBkKHXBxV4AfTR+MytuGrywJyqq+5sHzy4CDNgkaM803/CfDZKx0oaBZd8Egbu5RataQUbNmJI6U9T/MXDEqyAsC5UepuEcUNiBiaODEP+dVGjGV4ohdlu8CUoX7UlFUS5xcSkATqyJr2jMk+bABi3MKf1F+LiWvOS2ETxzz6OF32wMLPxLHgk2xRmC70xAA9f8KZ6Rs331Cl0EixKnzOs6udIXB9hv3cVJgVA9ydQ//3VK3hTy41XNe9OQ5S1P0X3Fmn95aaofobN25WITwGkkopPqWi6GppTuGy6N4qPY2rGAE=", + │ │ "thinking": "The user wants me to spawn a math-expert subagent to add 15 and 27 using the calculator tool. I should use the Agent tool with subagent_type set to \"math-expert\" and ask it to add 15 and 27.\n\nLet me create the appropriate prompt for the math-expert agent.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 1, + │ │ "prompt_cached_tokens": 19318, + │ │ "prompt_tokens": 19328, + │ │ "tokens": 19329 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "The result is 42. (15 + 27 = 42)", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 22906, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 127 + │ │ }, + │ │ "cache_creation_input_tokens": 127, + │ │ "cache_read_input_tokens": 22753, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 20, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: Add 15 and 27 together [task] + │ │ input: "Please use the calculator tool to add 15 and 27 together. Report the result." + │ │ output: [ + │ │ { + │ │ "text": "The result is 42. (15 + 27 = 42)", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "math-expert", + │ │ "claude_agent_sdk.description": "Add 15 and 27 together", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "mcp__calculator__calculator", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01JKWJX27jjQTVZ6zGBZETBo", + │ │ "claude_agent_sdk.total_tokens": 22772 + │ │ } + │ │ └── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ │ "role": "user" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "Add 15 and 27 together", + │ │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ │ "subagent_type": "math-expert" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ }, + │ │ "name": "mcp__calculator__calculator", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-haiku-4-5-20251001" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 8, + │ │ "prompt_cache_creation_tokens": 9514, + │ │ "prompt_cached_tokens": 13239, + │ │ "prompt_tokens": 22756, + │ │ "tokens": 22764 + │ │ } + │ │ └── tool: calculator/calculator [tool] + │ │ input: { + │ │ "a": 15, + │ │ "b": 27, + │ │ "operation": "add" + │ │ } + │ │ output: { + │ │ "content": [ + │ │ { + │ │ "text": "add(15, 27) = 42", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ │ "gen_ai.tool.call.id": "toolu_014XWmbx353UYpHZSZN82BU3", + │ │ "gen_ai.tool.name": "calculator", + │ │ "mcp.server": "calculator" + │ │ } + │ │ └── calculator-local-handler-add + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "Add 15 and 27 together", + │ "prompt": "Please use the calculator tool to add 15 and 27 together. Report the result.", + │ "subagent_type": "math-expert" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 15, + │ "b": 27, + │ "operation": "add" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "EtQCCmMIDRgCKkASrw/kOUsOJ9+lO04xzRcguHYCw8i89WyVqakHDanNnLEvGYNfkduGkxcbaUjQ/kljvn7aIOfSBaR2dDoc0lf3MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDN+GF8/AgJy5a3wkuxoM0x54pLnegBkIpHMGIjDFrroZD248rATm7/3KREHMlHZBwZxHASHdWFp2K/2E8+LN+BFdvuB1hviAjtSbrtIqngEQixeGsxK36fQVls3CquDgE2hhToqWKhJUT2mlW+tYDOMzoMLuQLuSpLg/RpA4Jz34KZv9RaJZRCTjbknVMtekwlUNFJD5HsncrihkEl+XoMtllHOfv5J9vQj52Zfku77xNfET+CRCgZo8YWyFtD/7MSnoX/528KO7/MmD5z7j8OQlDbCfsmnsYLiz9l6RFbsQ7bsQ9UFhwlvwb7a6NRgB", + │ "thinking": "Perfect! The math-expert subagent used the calculator tool to add 15 and 27 and got the result of 42. I should report this result to the user.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The math-expert subagent has completed the calculation using the calculator tool.\n\n**Result: 15 + 27 = 42**", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 269, + │ "prompt_cache_creation_tokens": 298, + │ "prompt_cached_tokens": 38636, + │ "prompt_tokens": 38942, + │ "tokens": 39211 + │ } + ├── claude-agent-subagent-built-in-tool-operation + │ metadata: { + │ "operation": "subagent-built-in-tool", + │ "testRunId": "" + │ } + │ └── Claude Agent [task] + │ input: "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent." + │ output: { + │ "content": [ + │ { + │ "text": "The echo agent has completed and sent a greeting message.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "allowedTools": [ + │ "Agent", + │ "Bash" + │ ], + │ "model": "claude-sonnet-4-5", + │ "num_turns": 2, + │ "permissionMode": "bypassPermissions", + │ "session_id": "" + │ } + │ ├── anthropic.messages.create [llm] + │ │ input: [ + │ │ { + │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "content": [ + │ │ { + │ │ "signature": "EtUGCmQIDRgCKkDxtVR9TJDmBhtkx+zbglgOXBnDvf7HmigKPWmisiAhyeGIMzRrSZVN4Oi0HqNAsZwgCKmHW90EWX5ou0OjIYrhMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy29yPv59LCEdPDLjoaDMM/bLL7dhTg3p0FKiIw9E3IYQG8JxM/zDiu0sjpzkNhMoxi3DBGFStJGPUFyYnN3Ilm+oT55NtnYFsCArq6Kp4FM2E7fN+FRpbPb1IXqCDJ6h5d3IyjS3oturfEtZofmtximwJwp0psinxfLD7V/YgJOYUfkOZwXNjhi/b9xk2OEozx0CL/VUZCcqt9whhiwpDFGehmFc+vnSNqffFjV7f/uKOuxcBmU447xdXBUwG6AD1vRXl/QkHRLLF0ni47HpFir0tOvGxX5kDN8hVD38zSehNwIEQp4XEDWuhjVQFpRfXx8Apnj5bFAkMgbR+cs3UPVM773/FbEijF2Ou/giC+xaoXVM47ShyJvKW8MDzHu1Whl2y8yvAgh8JNIUnI523zP6ukjbHNdiVx//cp7m152NvY9MWOM7YjLUGeA5ZQXtYcUHvIPkAaBI4vUAGd4mJmTlxWIlUELfzW5m3nxPQRk5rRYCGkSdLl8TabB73VKTp+Z4SRPJzTdRkc7LhqyZDHiU3WcctqXxwFP4hDNxnIG90qJcd/8z7VwnhDmhSHathUaITYyLt7FJ153a81SrCYYgE49o9Jyhiyfup6j+SIAAYrLWjuiygDRecDIVYyfhI7wrPnA3nBb35xLY6uDEwnH0sOKn5V4Ulyw5qGgB3sZ7EC8MGYTSWOYDJ8mmYodEXtDhOEaXel65B8mUjlpAJ8nfnAkLnko8az5Dat/wfA3z0mj3QCqBpbUkfpdOkyhjXVWONSjJp/3dMI6YppnXG20ceDYyq1U9y9tQOJedjTLQlK+sWzNNX6Ja7jACWWvVfxyEE428PfwZ4i3olnzvNp2tlf7VKMj9g3Nqgvrnesw9S+vMIKuJ0iy9LJK3I8TyU3aOdviozCZk8mkXErfurNijPqmUVF1QlXnfVqsOdlFX5IfJAgi+VjpQ1ue711I2rUze88qst7BDCVpeFyy3OjH1n6jLDiFZHRuazW3hgB", + │ │ "thinking": "The user is giving me a very specific instruction: I MUST call the Agent tool with subagent_type=\"echo\" and description \"echo greeting\". I should not call Bash myself, and I should not answer with text - I need to delegate to the echo sub-agent.\n\nLooking at the Agent tool description, the \"echo\" agent type \"Runs one bash echo and reports back\" and has access to the Bash tool.\n\nSo I need to invoke the Agent tool with:\n- subagent_type: \"echo\"\n- description: \"echo greeting\"\n- prompt: I need to provide a prompt for what the echo agent should do\n\nSince this is an echo agent that runs one bash echo, I should give it a prompt to echo a greeting message.", + │ │ "type": "thinking" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "caller": { + │ │ "type": "direct" + │ │ }, + │ │ "id": "", + │ │ "input": { + │ │ "description": "echo greeting", + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "subagent_type": "echo" + │ │ }, + │ │ "name": "Agent", + │ │ "type": "tool_use" + │ │ } + │ │ ], + │ │ "role": "assistant" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "model": "claude-sonnet-4-5-20250929" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 3, + │ │ "prompt_cached_tokens": 19218, + │ │ "prompt_tokens": 19228, + │ │ "tokens": 19231 + │ │ } + │ ├── tool: Agent [tool] + │ │ input: { + │ │ "description": "echo greeting", + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "subagent_type": "echo" + │ │ } + │ │ output: { + │ │ "agentId": "", + │ │ "content": [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ], + │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ "status": "completed", + │ │ "totalDurationMs": 0, + │ │ "totalTokens": 12972, + │ │ "totalToolUseCount": 1, + │ │ "usage": { + │ │ "cache_creation": { + │ │ "ephemeral_1h_input_tokens": 0, + │ │ "ephemeral_5m_input_tokens": 86 + │ │ }, + │ │ "cache_creation_input_tokens": 86, + │ │ "cache_read_input_tokens": 12876, + │ │ "inference_geo": "", + │ │ "input_tokens": 6, + │ │ "iterations": [], + │ │ "output_tokens": 4, + │ │ "server_tool_use": { + │ │ "web_fetch_requests": 0, + │ │ "web_search_requests": 0 + │ │ }, + │ │ "service_tier": "standard", + │ │ "speed": "standard" + │ │ } + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Agent", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + │ │ "gen_ai.tool.name": "Agent" + │ │ } + │ │ └── Agent: echo greeting [task] + │ │ input: "Run a bash echo command to output a greeting message." + │ │ output: [ + │ │ { + │ │ "text": "done", + │ │ "type": "text" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "claude_agent_sdk.agent_type": "echo", + │ │ "claude_agent_sdk.description": "Running Output a greeting message", + │ │ "claude_agent_sdk.duration_ms": 0, + │ │ "claude_agent_sdk.last_tool_name": "Bash", + │ │ "claude_agent_sdk.status": "completed", + │ │ "claude_agent_sdk.task_id": "", + │ │ "claude_agent_sdk.task_type": "local_agent", + │ │ "claude_agent_sdk.tool_use_count": 1, + │ │ "claude_agent_sdk.tool_use_id": "toolu_01GfQtHrDVn5VuNvuKZBv8Bf", + │ │ "claude_agent_sdk.total_tokens": 12938 + │ │ } + │ │ ├── anthropic.messages.create [llm] + │ │ │ input: [ + │ │ │ { + │ │ │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ │ │ "role": "user" + │ │ │ }, + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "description": "echo greeting", + │ │ │ "prompt": "Run a bash echo command to output a greeting message.", + │ │ │ "subagent_type": "echo" + │ │ │ }, + │ │ │ "name": "Agent", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ output: [ + │ │ │ { + │ │ │ "content": [ + │ │ │ { + │ │ │ "caller": { + │ │ │ "type": "direct" + │ │ │ }, + │ │ │ "id": "", + │ │ │ "input": { + │ │ │ "command": "echo \"hello\"", + │ │ │ "description": "Output a greeting message" + │ │ │ }, + │ │ │ "name": "Bash", + │ │ │ "type": "tool_use" + │ │ │ } + │ │ │ ], + │ │ │ "role": "assistant" + │ │ │ } + │ │ │ ] + │ │ │ metadata: { + │ │ │ "model": "claude-haiku-4-5-20251001" + │ │ │ } + │ │ │ metrics: { + │ │ │ "completion_tokens": 59, + │ │ │ "prompt_cache_creation_tokens": 12876, + │ │ │ "prompt_tokens": 12879, + │ │ │ "tokens": 12938 + │ │ │ } + │ │ └── tool: Bash [tool] + │ │ input: { + │ │ "command": "echo \"hello\"", + │ │ "description": "Output a greeting message" + │ │ } + │ │ output: { + │ │ "interrupted": false, + │ │ "isImage": false, + │ │ "noOutputExpected": false, + │ │ "stderr": "", + │ │ "stdout": "hello" + │ │ } + │ │ metadata: { + │ │ "claude_agent_sdk.cwd": "/e2e/scenarios/claude-agent-sdk-instrumentation/", + │ │ "claude_agent_sdk.raw_tool_name": "Bash", + │ │ "claude_agent_sdk.session_id": "", + │ │ "gen_ai.tool.call.id": "toolu_01PcrpA2E4F67kXt56n6W6er", + │ │ "gen_ai.tool.name": "Bash" + │ │ } + │ └── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You MUST call the Agent tool now with subagent_type=\"echo\" and description \"echo greeting\". Do not call Bash yourself. Do not answer with text yourself; delegate to the echo sub-agent.", + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "description": "echo greeting", + │ "prompt": "Run a bash echo command to output a greeting message.", + │ "subagent_type": "echo" + │ }, + │ "name": "Agent", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "command": "echo \"hello\"", + │ "description": "Output a greeting message" + │ }, + │ "name": "Bash", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "Eq4ICmQIDRgCKkDIBefT9K6Y1pICmXeUuPredEztqrbx9ShzVryB1SHZII1r0R7m70CdM1ft5bH/gsj48s+KEtPJflEUSdg7S2nzMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAEgy9Ik7serZBxmHIbjAaDNOjKhWq+7puWHSPziIwKLeDFvOXy374Pf5hhgsM7RVzPSmEKrFnv2Yi7K1xrC6CZnszI9LzukRy0lVbHAbKKvcGVK9PtEzmncnu9dCvoe/FOe/bgjGNULMaKHzVnY8Drgag2QeX//DbpOJ1IUgeILQvMSLVCP8DtaTUit0nxvoFZim4l73LEnA6SEWpZ/x4LnvfXVsA93nDBbBpHkbS+bjTlpWbLfmldufXcB22aAXhb1Y9KQW43OHcOMVDLLh2r8042zlrmBywEKofAen70hT3H6JvcRIK/hlwTDQaNpHiKqI8I2Gmy3R0L1rL79suIFK0eRpghFR9/B7+JeB8QmAFqn+Ihhby7hZ6kobGzwIh1OTOxsFUWdkEl1jsTD27yyqDE0sdiTHoD7Z8BTv1D5aDtfSo8HOTA/9G1M4QbnbawAe/Qf47DjzwhRnfXRvPi5dNK3YRJRpMWnrvez06owaKxcfleWSo6eLZDiKvmGYhfJlM/tjHO27PsQxB+vyIz0bR804XPOu1lGvMzMMEqHDUp3cU+WqR5OJaX0EJSkpjjgD/w+7ai5b3BcCiagc4YOPVZqqV/wsDSZc80uE2kqoj+KAv+tTygW+6xjYcX2HXbPc7xD49PCl72L1qc0fN/spTstDOo1xO4r3k9hjq7IWnIhCc1j0ct4GRAocYB2NzvVoixp7GRzdYMMUGYH4dw8GICE7DeTjFpVaOzAIJYzPDjXbCM0Yn/bRfehRAy5K0Tfn1Kz0e1LqONkaujXlFu7phzTPaxzB6woEg9vdL3V8GOlDggegJyUxJhBQWuOHnfII6RHmDdHRJKnRGNNrjystYg/3VR/xEGUnzxNjVNJeGcVU6f2cALKyyFnDnus/OQsOhsBuW6u0oomsP91g8SGJBVcyeaVLmiV6AM+fajVBmfedqUD00rNGLDMr9ICfkaTccILt3gG9ZjLVKWBIhBTqOUNYaU9pkinikD/bJnYBjIhUJ9drAl3Ftk1gyisPCguhN96RCz7pIIGVg2kE8RbcRlRsIZRCEu1bcDE73ZBL93COGtOUnldyG39PWbPC/jDh10mamHjfSccwB3UNT4TEdMXJQYFDOoDALXtS+6sWEzLO7JXjOxyXY0Ic/7w9Kc32Ui/82tjSMfmUWfj+mY/ecVSWuG2rcCN1DNXM/ppnE2CuRVp9MwZgzTtAKpKI6pnX/P+sEbixkipya4mjX6xpjrQfC3YiTGeYyPFcLZHwylS6JKGj1evVcb9/8QFNXKJ9tu73r56sYAQ==", + │ "thinking": "The echo agent has completed. According to the instructions, when an agent is done, it returns a single message back to me, and the result is not visible to the user. I should send a text message back to the user with a concise summary of the result.\n\nHowever, I notice the function result doesn't actually show me what the echo agent said or did - it just shows it completed with agent ID aa78b663afc5637a9.\n\nBut wait, looking more carefully at the instructions: \"When an agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\"\n\nThe result here seems to just be \"done\" without any actual output. This is a bit unusual. Let me respond to the user that the echo agent has completed its task.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "text": "The echo agent has completed and sent a greeting message.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-sonnet-4-5-20250929" + │ } + │ metrics: { + │ "completion_tokens": 430, + │ "prompt_cache_creation_tokens": 362, + │ "prompt_cached_tokens": 38436, + │ "prompt_tokens": 38806, + │ "tokens": 39236 + │ } + └── claude-agent-failure-operation + metadata: { + "operation": "failure", + "testRunId": "" + } + └── Claude Agent [task] + input: "Use the calculator tool to divide 2 by 0. Do not recover from the error." + output: { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + metadata: { + "model": "claude-haiku-4-5", + "num_turns": 2, + "permissionMode": "bypassPermissions", + "session_id": "" + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "content": [ + │ { + │ "signature": "ErYECmMIDRgCKkBBXV54rxI9PiQXBXUh7+BZnW7Ej/LcxR6xa+6w53+YjsNNCH5pPtVLZs2SgMGo0dP1ekxe1LLBZ6VB7+3LNkItMhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDNAhx6f1XtDIFDLdZhoMescY53N4oymlYKUwIjAuB+LUXhWcDodizE9nhNjhMi75aOPKOG74BdiAl45os3YoZW4K8Sm+p5GT1VkhtOIqgAMfS3JhW1A3ktD7tgpUF9hQi7jNPbpsqiyg9ScbYw10ATYORqJ/KJIeJyglFwuVBvNQlpDsUTHT07M8bKv5sP/MPR0hrFSQPF+6dSL5hyitpFfaUzGgb+lXegLNNMTTvhZEw1/9a/hNAITW8YuJE5RJeIaq56CJ/SwakrwAk8KxyEOVHYvKUiStaTAdYe8LGcH8Yy6ToPwocTmg8aV97291AqFTXBPyAAAgLw1Ix5uuzQ7hxZ2xnpZYUCD5uSBcbn92TFTFJSYXllHgpoy81DWbi/mSS6GQq3G0enVuvMXtMqHUfR40doHF7JW3VDGKVqR1kkAlggKDobJkc59wmeLeGEM0z/Goo10U2jus+CoPXYh/pHmmor7Lp8eDNfNJ2SEgmVNsBGyr8VT9vFg/GhOCQlpxMvZd5zRILdDwECh+t51bOVFG3yGbREjuLI5tci97yBLuamezYkeA1bIXZYgWp7zBlDzhRXDhSwZ3bPoHx71j5tsZIz3g7XVwufHEH6sYAQ==", + │ "thinking": "The user is asking me to use the calculator tool to divide 2 by 0, and explicitly tells me not to recover from the error. This is testing what happens when I attempt an invalid mathematical operation.\n\nLet me use the calculator tool with:\n- operation: \"divide\"\n- a: 2\n- b: 0\n\nThe user has explicitly asked me to attempt this and not recover from any error that occurs.", + │ "type": "thinking" + │ } + │ ], + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "caller": { + │ "type": "direct" + │ }, + │ "id": "", + │ "input": { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ }, + │ "name": "mcp__calculator__calculator", + │ "type": "tool_use" + │ } + │ ], + │ "role": "assistant" + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5-20251001" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 19291, + │ "prompt_tokens": 19301, + │ "tokens": 19309 + │ } + │ └── tool: calculator/calculator [tool] + │ input: { + │ "a": 2, + │ "b": 0, + │ "operation": "divide" + │ } + │ metadata: { + │ "claude_agent_sdk.raw_tool_name": "mcp__calculator__calculator", + │ "gen_ai.tool.call.id": "toolu_01HM53Dgu54gnzQEZMp744Ua", + │ "gen_ai.tool.name": "calculator", + │ "mcp.server": "calculator" + │ } + │ error: "division by zero" + │ └── calculator-local-handler-divide + │ error: "division by zero\n\nError: division by zero\n at result.name (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at /js/dist/index.js:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.js:0:0)\n at withCurrent (/js/dist/index.js:0:0)\n at /js/dist/index.js:0:0\n at runCatchFinally (/js/dist/index.js:0:0)\n at traced (/js/dist/index.js:0:0)\n at (/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.impl.mjs:0:0)\n at runHandler (/js/dist/index.js:0:0)" + └── anthropic.messages.create [llm] + input: [ + { + "content": "Use the calculator tool to divide 2 by 0. Do not recover from the error.", + "role": "user" + }, + { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "a": 2, + "b": 0, + "operation": "divide" + }, + "name": "mcp__calculator__calculator", + "type": "tool_use" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "signature": "Eq0DCmMIDRgCKkBycB6lZSvTw0Vn6nK+WvTd6RCisqfmwFHqfGlCUWt5cNemWVLFN/rO3NviniXxZhPj5trzQ8iHoafOdyoGDTT8MhljbGF1ZGUtaGFpa3UtNC01LTIwMjUxMDAxOAASDLHQaayWIYF0q6r7shoM9Ua7OzUkDWfKdNOIIjC4VCnTxXOXt4KbD7C4GbGjemoqargi2hPLax3rVbTeTXo1c0k+oNeyQUARCwe0dQgq9wHhDg3CdkVLQZHLyGcTgIDTuuYDDqa5QodSzTlc0KpAF8FKlzYYAD+j83xLSYRu9XmCHzntbUzZkvuqrIFyFTFNf2vTdgI+xrZo+zf9vXJSkmwk1f3PwFycVIkYk2yxokmqN71R+ZqTBpqrAAfAsWYWl4d+la8XFZ9biMYAYsJhHeZL7G763G9kw/ClUGvNxuQ23f8eBV6IVvJ0Ee8K+o9BmzLJb8jaIXfQyyLmLewrw9fS1vutUwcXrLF41Xoj926Log3MHPJjhyLRSPWgxaRJ1pIE8VvuxZVgAuo3WCxuZC9SyvkrcSXOmS9YlmxLPbFR74JVo0AyGAE=", + "thinking": "The calculator tool returned a division by zero error, as expected. The user specifically asked me not to recover from the error, so I should just present this error result without attempting to handle it or provide an alternative.", + "type": "thinking" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "text": "The operation resulted in a **division by zero** error.", + "type": "text" + } + ], + "role": "assistant" + } + ] + metadata: { + "model": "claude-haiku-4-5-20251001" + } + metrics: { + "completion_tokens": 246, + "prompt_cache_creation_tokens": 220, + "prompt_cached_tokens": 38582, + "prompt_tokens": 38810, + "tokens": 39056 + } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json b/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json deleted file mode 100644 index 5d13ceeaa..000000000 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/__snapshots__/claude-agent-sdk-v0.2.81.span-events.json +++ /dev/null @@ -1,316 +0,0 @@ -{ - "async_prompt": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "async-prompt" - }, - "metric_keys": [], - "name": "claude-agent-async-prompt-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Part 1", - "Part 2" - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "basic": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to multiply 15 by 7. Do not answer from memory." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "basic" - }, - "metric_keys": [], - "name": "claude-agent-basic-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "failure": { - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Use the calculator tool to divide 2 by 0. Do not recover from the error." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "failure" - }, - "metric_keys": [], - "name": "claude-agent-failure-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "error": "division by zero", - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "claude-agent-sdk-traces" - }, - "metric_keys": [], - "name": "claude-agent-sdk-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "subagent": { - "handoff_tool": { - "has_input": true, - "has_output": false, - "metadata": { - "gen_ai.tool.name": "Agent" - }, - "metric_keys": [], - "name": "tool: Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - "llm": { - "has_input": true, - "has_output": true, - "input_contents": [ - "Spawn a math-expert subagent to add 15 and 27 using the calculator tool. Report the result. Do not solve it yourself." - ], - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "anthropic.messages.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "nested_task": { - "has_input": false, - "has_output": false, - "metadata": { - "claude_agent_sdk.agent_type": "math-expert", - "claude_agent_sdk.description": "", - "claude_agent_sdk.task_id": "", - "claude_agent_sdk.task_type": "local_agent", - "claude_agent_sdk.tool_use_id": "" - }, - "metric_keys": [], - "name": "Agent: ", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "subagent" - }, - "metric_keys": [], - "name": "claude-agent-subagent-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task_root": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "claude-haiku-4-5" - }, - "metric_keys": [], - "name": "Claude Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.name": "calculator", - "mcp.server": "calculator" - }, - "metric_keys": [], - "name": "tool: calculator/calculator", - "root_span_id": "", - "span_id": "", - "type": "tool" - } - } -} diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts index 89c3b585d..b5f07ac18 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts @@ -1,11 +1,7 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { effectiveScenarioTimeoutMs, withScenarioHarness, @@ -16,6 +12,11 @@ import { findChildSpans, findLatestSpan, } from "../../helpers/trace-selectors"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -277,7 +278,44 @@ function findOperationTaskRoot( return findChildSpans(events, "Claude Agent", operation?.span.id).at(-1); } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function spanTreeEntry( + event: CapturedLogEvent | undefined, + overrides?: { + metadata?: Json; + name?: string | null; + omitSpanParents?: boolean; + }, +): SpanTreeEntry | undefined { + if (!event) { + return undefined; + } + + const summary = summarizeSpan(event, overrides) as Record; + const { + has_input: _hasInput, + has_output: _hasOutput, + input_contents: inputContents, + metric_keys: metricKeys, + name, + root_span_id: _rootSpanId, + span_id: _spanId, + span_parents: _spanParents, + ...summaryFields + } = summary; + + return { + event, + fields: { + ...spanTreeFields(event), + ...summaryFields, + ...(inputContents === undefined ? {} : { input: inputContents }), + ...(metricKeys === undefined ? {} : { metrics: { keys: metricKeys } }), + }, + name: typeof name === "string" ? name : undefined, + }; +} + +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { const root = findLatestSpan(events, ROOT_NAME); const basicOperation = findLatestSpan(events, "claude-agent-basic-operation"); const asyncPromptOperation = findLatestSpan( @@ -357,34 +395,26 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { findToolSpanByLocalHandler(events, "calculator-local-handler-divide") ?? findToolSpanByOperation(events, "divide"); - return normalizeForSnapshot({ - async_prompt: { - llm: summarizeSpan(asyncPromptLlm), - operation: summarizeSpan(asyncPromptOperation), - task: summarizeSpan(asyncPromptTask), - }, - basic: { - llm: summarizeSpan(basicLlm), - operation: summarizeSpan(basicOperation), - task: summarizeSpan(basicTask), - tool: summarizeSpan(basicTool), - }, - failure: { - llm: summarizeSpan(failureLlm), - operation: summarizeSpan(failureOperation), - task: summarizeSpan(failureTask), - tool: summarizeSpan(failureTool), - }, - root: summarizeSpan(root), - subagent: { - handoff_tool: summarizeSpan(subAgentHandoffTool), - llm: summarizeSpan(subAgentLlm), - nested_task: summarizeSpan(subAgentTask), - operation: summarizeSpan(subAgentOperation), - task_root: summarizeSpan(subAgentTaskRoot), - tool: summarizeSpan(subAgentTool, { omitSpanParents: true }), - }, - } as Json); + return [ + spanTreeEntry(root), + spanTreeEntry(basicOperation), + spanTreeEntry(basicTask), + spanTreeEntry(basicLlm), + spanTreeEntry(basicTool), + spanTreeEntry(asyncPromptOperation), + spanTreeEntry(asyncPromptTask), + spanTreeEntry(asyncPromptLlm), + spanTreeEntry(subAgentOperation), + spanTreeEntry(subAgentTaskRoot), + spanTreeEntry(subAgentHandoffTool), + spanTreeEntry(subAgentTask), + spanTreeEntry(subAgentLlm), + spanTreeEntry(subAgentTool, { omitSpanParents: true }), + spanTreeEntry(failureOperation), + spanTreeEntry(failureTask), + spanTreeEntry(failureLlm), + spanTreeEntry(failureTool), + ].filter((entry): entry is SpanTreeEntry => entry !== undefined); } export function defineClaudeAgentSDKInstrumentationAssertions(options: { @@ -398,7 +428,7 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { }): void { const snapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); const testConfig = { @@ -691,11 +721,8 @@ export function defineClaudeAgentSDKInstrumentationAssertions(options: { } }); - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events)), - snapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, snapshotPath); }); }); } diff --git a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.test.ts b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.test.ts index 1786d1858..242e33726 100644 --- a/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.test.ts +++ b/e2e/scenarios/claude-agent-sdk-instrumentation/scenario.test.ts @@ -63,7 +63,7 @@ describe.concurrent("wrapped instrumentation", () => { timeoutMs: TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-wrapped`, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); @@ -90,7 +90,7 @@ describe.concurrent("auto-hook instrumentation", () => { timeoutMs: TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-auto-hook`, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-events.json deleted file mode 100644 index cf928fca7..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-events.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.json new file mode 100644 index 000000000..e3c982224 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.json @@ -0,0 +1,289 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.txt new file mode 100644 index 000000000..cbd405b61 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0-wrapped.span-tree.txt @@ -0,0 +1,229 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-events.json deleted file mode 100644 index 1ab342dc3..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-events.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.json new file mode 100644 index 000000000..050aa36bc --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.json @@ -0,0 +1,174 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.txt new file mode 100644 index 000000000..0c9a2a262 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-14-0.span-tree.txt @@ -0,0 +1,134 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-events.json deleted file mode 100644 index cf928fca7..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-events.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.json new file mode 100644 index 000000000..e3c982224 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.json @@ -0,0 +1,289 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.txt new file mode 100644 index 000000000..cbd405b61 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0-wrapped.span-tree.txt @@ -0,0 +1,229 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-events.json deleted file mode 100644 index 1ab342dc3..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-events.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.json new file mode 100644 index 000000000..050aa36bc --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.json @@ -0,0 +1,174 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "response_type": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.txt new file mode 100644 index 000000000..0c9a2a262 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-20-0.span-tree.txt @@ -0,0 +1,134 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "response_type": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-events.json deleted file mode 100644 index cf928fca7..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-events.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.json new file mode 100644 index 000000000..8aa00e89f --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.json @@ -0,0 +1,289 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.txt new file mode 100644 index 000000000..7a20b8ecf --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0-wrapped.span-tree.txt @@ -0,0 +1,229 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-events.json deleted file mode 100644 index 1ab342dc3..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-events.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.json new file mode 100644 index 000000000..85ee72c06 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.json @@ -0,0 +1,174 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.txt new file mode 100644 index 000000000..9266d8941 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-21-0.span-tree.txt @@ -0,0 +1,134 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-events.json deleted file mode 100644 index cf928fca7..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-events.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": false, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.json new file mode 100644 index 000000000..8aa00e89f --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.json @@ -0,0 +1,289 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.txt new file mode 100644 index 000000000..7a20b8ecf --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7-wrapped.span-tree.txt @@ -0,0 +1,229 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ metadata: { + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-events.json deleted file mode 100644 index 1ab342dc3..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-events.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-v2-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-v2-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-v2-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-english-v3.0", - "provider": "cohere" - }, - "metric_keys": [ - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-v2-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-english-v3.0", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.json new file mode 100644 index 000000000..85ee72c06 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.json @@ -0,0 +1,174 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-v2-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-v2-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-v2-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1024 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-english-v3.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-v2-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.txt new file mode 100644 index 000000000..9266d8941 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v7.span-tree.txt @@ -0,0 +1,134 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-v2-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-v2-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-english-v3.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "prompt_tokens": 3 + │ } + └── cohere-v2-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.9980122 + }, + { + "index": 2, + "relevance_score": 0.00015720345 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-english-v3.0", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-events.json deleted file mode 100644 index 9a11e3ffa..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-events.json +++ /dev/null @@ -1,159 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-v4.0", - "provider": "cohere" - }, - "metric_keys": [ - "image_tokens", - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-v3.5", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.json new file mode 100644 index 000000000..52a342d14 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.json @@ -0,0 +1,175 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1536 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-v4.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "image_tokens": 0, + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.8657814 + }, + { + "index": 1, + "relevance_score": 0.09116359 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-v3.5", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.txt new file mode 100644 index 000000000..58a40f35d --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8-wrapped.span-tree.txt @@ -0,0 +1,135 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-v4.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "image_tokens": 0, + │ "prompt_tokens": 3 + │ } + └── cohere-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.8657814 + }, + { + "index": 1, + "relevance_score": 0.09116359 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-v3.5", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-events.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-events.json deleted file mode 100644 index 9a11e3ffa..000000000 --- a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-events.json +++ /dev/null @@ -1,159 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cohere-instrumentation" - }, - "metric_keys": [], - "name": "cohere-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "cohere-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "cohere-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "command-a-03-2025", - "provider": "cohere" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "cohere.chatStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "cohere-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "inputType": "search_document", - "model": "embed-v4.0", - "provider": "cohere" - }, - "metric_keys": [ - "image_tokens", - "prompt_tokens" - ], - "name": "cohere.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "cohere-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-v3.5", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "search_units" - ], - "name": "cohere.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.json b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.json new file mode 100644 index 000000000..52a342d14 --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.json @@ -0,0 +1,175 @@ +{ + "span_tree": [ + { + "name": "cohere-instrumentation-root", + "type": "task", + "children": [ + { + "name": "cohere-chat-operation", + "children": [ + { + "name": "cohere.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "metadata": { + "finishReason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "cohere-chat-stream-operation", + "children": [ + { + "name": "cohere.chatStream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": { + "content": "STREAM.", + "role": "assistant" + }, + "metadata": { + "finish_reason": "COMPLETE", + "id": "", + "maxTokens": 32, + "model": "command-a-03-2025", + "provider": "cohere", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "cohere-embed-operation", + "children": [ + { + "name": "cohere.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing" + ], + "output": { + "embedding_length": 1536 + }, + "metadata": { + "api_version": "2", + "embeddingTypes": [ + "float" + ], + "id": "", + "inputType": "search_document", + "model": "embed-v4.0", + "provider": "cohere", + "responseType": "embeddings_by_type" + }, + "metrics": { + "image_tokens": 0, + "prompt_tokens": 3 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "cohere-rerank-operation", + "children": [ + { + "name": "cohere.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + }, + "output": [ + { + "index": 0, + "relevance_score": 0.8657814 + }, + { + "index": 1, + "relevance_score": 0.09116359 + } + ], + "metadata": { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-v3.5", + "provider": "cohere", + "topN": 2 + }, + "metrics": { + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.txt b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.txt new file mode 100644 index 000000000..58a40f35d --- /dev/null +++ b/e2e/scenarios/cohere-instrumentation/__snapshots__/cohere-v8.span-tree.txt @@ -0,0 +1,135 @@ +span_tree: +└── cohere-instrumentation-root [task] + metadata: { + "scenario": "cohere-instrumentation", + "testRunId": "" + } + ├── cohere-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── cohere.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": [ + │ { + │ "text": "OK.", + │ "type": "text" + │ } + │ ], + │ "role": "assistant" + │ } + │ metadata: { + │ "finishReason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── cohere.chatStream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "content": "STREAM.", + │ "role": "assistant" + │ } + │ metadata: { + │ "finish_reason": "COMPLETE", + │ "id": "", + │ "maxTokens": 32, + │ "model": "command-a-03-2025", + │ "provider": "cohere", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 7 + │ } + ├── cohere-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── cohere.embed [llm] + │ input: [ + │ "braintrust tracing" + │ ] + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "api_version": "2", + │ "embeddingTypes": [ + │ "float" + │ ], + │ "id": "", + │ "inputType": "search_document", + │ "model": "embed-v4.0", + │ "provider": "cohere", + │ "responseType": "embeddings_by_type" + │ } + │ metrics: { + │ "image_tokens": 0, + │ "prompt_tokens": 3 + │ } + └── cohere-rerank-operation + metadata: { + "operation": "rerank", + "testRunId": "" + } + └── cohere.rerank [llm] + input: { + "documents": [ + "Paris is the capital city of France.", + "Vienna is the capital city of Austria.", + "Canberra is the capital city of Australia." + ], + "query": "What is the capital of France?" + } + output: [ + { + "index": 0, + "relevance_score": 0.8657814 + }, + { + "index": 1, + "relevance_score": 0.09116359 + } + ] + metadata: { + "api_version": "2", + "document_count": 3, + "id": "", + "model": "rerank-v3.5", + "provider": "cohere", + "topN": 2 + } + metrics: { + "search_units": 1 + } diff --git a/e2e/scenarios/cohere-instrumentation/assertions.ts b/e2e/scenarios/cohere-instrumentation/assertions.ts index 3d44e9538..6e6e9ce5d 100644 --- a/e2e/scenarios/cohere-instrumentation/assertions.ts +++ b/e2e/scenarios/cohere-instrumentation/assertions.ts @@ -1,17 +1,12 @@ import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; type RunCohereScenario = (harness: { @@ -54,7 +49,7 @@ function buildSpanSummary( events: CapturedLogEvent[], supportsThinking: boolean, useV2Namespace: boolean, -): Json { +): CapturedLogEvent[] { const chatOperation = findLatestSpan( events, getOperationName("chat", { useV2Namespace }), @@ -101,18 +96,7 @@ function buildSpanSummary( ); } - return summaryEvents.map((event) => - summarizeWrapperContract(event!, [ - "document_count", - "inputType", - "model", - "operation", - "provider", - "scenario", - "thinking", - "topN", - ]), - ) as Json; + return summaryEvents.map((event) => event!); } export function defineCohereInstrumentationAssertions(options: { @@ -127,7 +111,7 @@ export function defineCohereInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -287,17 +271,8 @@ export function defineCohereInstrumentationAssertions(options: { }); }); - test("matches span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildSpanSummary( - events, - options.supportsThinking, - options.useV2Namespace ?? false, - ), - ), - spanSnapshotPath, - ); + test("matches span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-events.json b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-events.json deleted file mode 100644 index 8365c2dfa..000000000 --- a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-events.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "conversation": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "resume-conversation" - }, - "metric_keys": [], - "name": "cursor-sdk-resume-conversation-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "prompt": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "prompt" - }, - "metric_keys": [], - "name": "cursor-sdk-prompt-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "Agent.prompt", - "cursor_sdk.run_id": "", - "cursor_sdk.runtime": "local", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cursor-sdk-instrumentation" - }, - "metric_keys": [], - "name": "cursor-sdk-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "stream": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "cursor-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "subagent_task": null, - "subagent_tool": null, - "task": { - "has_input": true, - "has_output": false, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": null - }, - "wait": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "wait" - }, - "metric_keys": [], - "name": "cursor-sdk-wait-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": false, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - } -} diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.json b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.json new file mode 100644 index 000000000..05ee68fd7 --- /dev/null +++ b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.json @@ -0,0 +1,135 @@ +{ + "span_tree": [ + { + "name": "cursor-sdk-root", + "type": "task", + "children": [ + { + "name": "cursor-sdk-prompt-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_PROMPT_OK. Do not modify files.", + "output": { + "durationMs": 0, + "id": "", + "model": { + "id": "" + }, + "status": "error" + }, + "metadata": { + "cursor_sdk.duration_ms": 0, + "cursor_sdk.local.cwd": "/e2e/scenarios/cursor-sdk-instrumentation/", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "Agent.prompt", + "cursor_sdk.run_id": "run-", + "cursor_sdk.runtime": "local", + "cursor_sdk.status": "error", + "cursor_sdk.version": "1.0.12", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "prompt", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-stream-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "First use the reviewer subagent to confirm exactly CURSOR_SUBAGENT_OK. Then run the shell command `printf cursor_tool_ok` and report the output. Do not edit files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.duration_ms": 0, + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-wait-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_WAIT_OK. Do not modify files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.duration_ms": 0, + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "wait", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-resume-conversation-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_CONVERSATION_OK. Do not modify files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "resume-conversation", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cursor-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.txt b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.txt new file mode 100644 index 000000000..231f2c85d --- /dev/null +++ b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-auto-hook.span-tree.txt @@ -0,0 +1,95 @@ +span_tree: +└── cursor-sdk-root [task] + metadata: { + "scenario": "cursor-sdk-instrumentation", + "testRunId": "" + } + ├── cursor-sdk-prompt-operation + │ metadata: { + │ "operation": "prompt", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "Reply with exactly: CURSOR_PROMPT_OK. Do not modify files." + │ output: { + │ "durationMs": 0, + │ "id": "", + │ "model": { + │ "id": "" + │ }, + │ "status": "error" + │ } + │ metadata: { + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.local.cwd": "/e2e/scenarios/cursor-sdk-instrumentation/", + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "Agent.prompt", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.runtime": "local", + │ "cursor_sdk.status": "error", + │ "cursor_sdk.version": "1.0.12", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── cursor-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "First use the reviewer subagent to confirm exactly CURSOR_SUBAGENT_OK. Then run the shell command `printf cursor_tool_ok` and report the output. Do not edit files." + │ metadata: { + │ "cursor_sdk.agent_id": "agent-", + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "agent.send", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.status": "error", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── cursor-sdk-wait-operation + │ metadata: { + │ "operation": "wait", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "Reply with exactly: CURSOR_WAIT_OK. Do not modify files." + │ metadata: { + │ "cursor_sdk.agent_id": "agent-", + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "agent.send", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.status": "error", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── cursor-sdk-resume-conversation-operation + metadata: { + "operation": "resume-conversation", + "testRunId": "" + } + └── Cursor Agent [task] + input: "Reply with exactly: CURSOR_CONVERSATION_OK. Do not modify files." + metadata: { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-events.json b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-events.json deleted file mode 100644 index 8365c2dfa..000000000 --- a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-events.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "conversation": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "resume-conversation" - }, - "metric_keys": [], - "name": "cursor-sdk-resume-conversation-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "prompt": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "prompt" - }, - "metric_keys": [], - "name": "cursor-sdk-prompt-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "Agent.prompt", - "cursor_sdk.run_id": "", - "cursor_sdk.runtime": "local", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "cursor-sdk-instrumentation" - }, - "metric_keys": [], - "name": "cursor-sdk-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "stream": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "cursor-sdk-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "subagent_task": null, - "subagent_tool": null, - "task": { - "has_input": true, - "has_output": false, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": null - }, - "wait": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "wait" - }, - "metric_keys": [], - "name": "cursor-sdk-wait-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": false, - "metadata": { - "cursor_sdk.agent_id": "", - "cursor_sdk.model": "composer-2", - "cursor_sdk.operation": "agent.send", - "cursor_sdk.run_id": "", - "cursor_sdk.status": "error", - "model": "composer-2", - "provider": "cursor" - }, - "metric_keys": [ - "duration" - ], - "name": "Cursor Agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - } -} diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.json b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.json new file mode 100644 index 000000000..20ac33ba5 --- /dev/null +++ b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.json @@ -0,0 +1,134 @@ +{ + "span_tree": [ + { + "name": "cursor-sdk-root", + "type": "task", + "children": [ + { + "name": "cursor-sdk-prompt-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_PROMPT_OK. Do not modify files.", + "output": { + "durationMs": 0, + "id": "", + "model": { + "id": "" + }, + "status": "error" + }, + "metadata": { + "cursor_sdk.duration_ms": 0, + "cursor_sdk.local.cwd": "/e2e/scenarios/cursor-sdk-instrumentation/", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "Agent.prompt", + "cursor_sdk.run_id": "run-", + "cursor_sdk.runtime": "local", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "prompt", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-stream-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "First use the reviewer subagent to confirm exactly CURSOR_SUBAGENT_OK. Then run the shell command `printf cursor_tool_ok` and report the output. Do not edit files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.duration_ms": 0, + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-wait-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_WAIT_OK. Do not modify files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.duration_ms": 0, + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "wait", + "testRunId": "" + } + }, + { + "name": "cursor-sdk-resume-conversation-operation", + "children": [ + { + "name": "Cursor Agent", + "type": "task", + "children": [], + "input": "Reply with exactly: CURSOR_CONVERSATION_OK. Do not modify files.", + "metadata": { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "resume-conversation", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "cursor-sdk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.txt b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.txt new file mode 100644 index 000000000..4172c2799 --- /dev/null +++ b/e2e/scenarios/cursor-sdk-instrumentation/__snapshots__/cursor-sdk-v1-wrapped.span-tree.txt @@ -0,0 +1,94 @@ +span_tree: +└── cursor-sdk-root [task] + metadata: { + "scenario": "cursor-sdk-instrumentation", + "testRunId": "" + } + ├── cursor-sdk-prompt-operation + │ metadata: { + │ "operation": "prompt", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "Reply with exactly: CURSOR_PROMPT_OK. Do not modify files." + │ output: { + │ "durationMs": 0, + │ "id": "", + │ "model": { + │ "id": "" + │ }, + │ "status": "error" + │ } + │ metadata: { + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.local.cwd": "/e2e/scenarios/cursor-sdk-instrumentation/", + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "Agent.prompt", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.runtime": "local", + │ "cursor_sdk.status": "error", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── cursor-sdk-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "First use the reviewer subagent to confirm exactly CURSOR_SUBAGENT_OK. Then run the shell command `printf cursor_tool_ok` and report the output. Do not edit files." + │ metadata: { + │ "cursor_sdk.agent_id": "agent-", + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "agent.send", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.status": "error", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── cursor-sdk-wait-operation + │ metadata: { + │ "operation": "wait", + │ "testRunId": "" + │ } + │ └── Cursor Agent [task] + │ input: "Reply with exactly: CURSOR_WAIT_OK. Do not modify files." + │ metadata: { + │ "cursor_sdk.agent_id": "agent-", + │ "cursor_sdk.duration_ms": 0, + │ "cursor_sdk.model": "composer-2", + │ "cursor_sdk.operation": "agent.send", + │ "cursor_sdk.run_id": "run-", + │ "cursor_sdk.status": "error", + │ "model": "composer-2", + │ "provider": "cursor" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── cursor-sdk-resume-conversation-operation + metadata: { + "operation": "resume-conversation", + "testRunId": "" + } + └── Cursor Agent [task] + input: "Reply with exactly: CURSOR_CONVERSATION_OK. Do not modify files." + metadata: { + "cursor_sdk.agent_id": "agent-", + "cursor_sdk.model": "composer-2", + "cursor_sdk.operation": "agent.send", + "cursor_sdk.run_id": "run-", + "cursor_sdk.status": "error", + "model": "composer-2", + "provider": "cursor" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/cursor-sdk-instrumentation/assertions.ts b/e2e/scenarios/cursor-sdk-instrumentation/assertions.ts index d110efc02..766853427 100644 --- a/e2e/scenarios/cursor-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/cursor-sdk-instrumentation/assertions.ts @@ -1,11 +1,6 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { effectiveScenarioTimeoutMs, withScenarioHarness, @@ -16,7 +11,12 @@ import { findChildSpans, findLatestSpan, } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, + type SpanTreeFields, +} from "../../helpers/span-tree"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; type RunCursorSDKScenario = (harness: { @@ -51,30 +51,30 @@ const METADATA_KEYS = [ "cursor_sdk.tool.status", ] as const; -function summarizeSpan(event: CapturedLogEvent | undefined): Json { - if (!event) { - return null; - } - const summary = summarizeWrapperContract(event, [...METADATA_KEYS]) as Record< - string, - Json - >; - if (summary.metadata && typeof summary.metadata === "object") { - const metadata = summary.metadata as Record; - if (typeof metadata["cursor_sdk.agent_id"] === "string") { - metadata["cursor_sdk.agent_id"] = ""; - } - if (typeof metadata["cursor_sdk.run_id"] === "string") { - metadata["cursor_sdk.run_id"] = ""; - } - } - if (typeof event.row.error === "string") { - summary.error = event.row.error; +function snapshotFields(event: CapturedLogEvent): SpanTreeFields { + const fields = spanTreeFields(event); + const metadata = + fields.metadata && + typeof fields.metadata === "object" && + !Array.isArray(fields.metadata) + ? Object.fromEntries( + Object.entries(fields.metadata).filter(([key]) => + METADATA_KEYS.includes(key as (typeof METADATA_KEYS)[number]), + ), + ) + : undefined; + + if (metadata && typeof metadata["cursor_sdk.agent_id"] === "string") { + metadata["cursor_sdk.agent_id"] = ""; } - if (typeof summary.name === "string" && summary.name.startsWith("Agent:")) { - summary.name = "Agent: "; + if (metadata && typeof metadata["cursor_sdk.run_id"] === "string") { + metadata["cursor_sdk.run_id"] = ""; } - return summary; + + return { + ...fields, + metadata, + }; } function findOperation(events: CapturedLogEvent[], name: string) { @@ -126,7 +126,7 @@ function outputText(event: CapturedLogEvent | undefined): string { return typeof event?.output === "string" ? event.output : ""; } -function summarize(events: CapturedLogEvent[]): Json { +function summarize(events: CapturedLogEvent[]): SpanTreeEntry[] { const promptTask = findCursorTask(events, "cursor-sdk-prompt-operation"); const streamTask = findCursorTask(events, "cursor-sdk-stream-operation"); const waitTask = findCursorTask(events, "cursor-sdk-wait-operation"); @@ -138,36 +138,32 @@ function summarize(events: CapturedLogEvent[]): Json { const subagentTool = findSubagentTool(events, streamTask?.span.id); const subagentTask = findSubagentTask(events, subagentTool?.span.id); - return normalizeForSnapshot({ - conversation: { - operation: summarizeSpan( - findOperation(events, "cursor-sdk-resume-conversation-operation"), - ), - task: summarizeSpan(conversationTask), - }, - prompt: { - operation: summarizeSpan( - findOperation(events, "cursor-sdk-prompt-operation"), - ), - task: summarizeSpan(promptTask), - }, - root: summarizeSpan(findLatestSpan(events, ROOT_NAME)), - stream: { - operation: summarizeSpan( - findOperation(events, "cursor-sdk-stream-operation"), - ), - subagent_task: summarizeSpan(subagentTask), - subagent_tool: summarizeSpan(subagentTool), - task: summarizeSpan(streamTask), - tool: summarizeSpan(tool), - }, - wait: { - operation: summarizeSpan( - findOperation(events, "cursor-sdk-wait-operation"), - ), - task: summarizeSpan(waitTask), - }, - } as Json); + return [ + findLatestSpan(events, ROOT_NAME), + findOperation(events, "cursor-sdk-prompt-operation"), + promptTask, + findOperation(events, "cursor-sdk-stream-operation"), + streamTask, + tool, + subagentTool, + subagentTask, + findOperation(events, "cursor-sdk-wait-operation"), + waitTask, + findOperation(events, "cursor-sdk-resume-conversation-operation"), + conversationTask, + ].flatMap((event) => + event + ? [ + { + event, + fields: snapshotFields(event), + name: event.span.name?.startsWith("Agent:") + ? "Agent: " + : undefined, + }, + ] + : [], + ); } export function defineCursorSDKInstrumentationAssertions(options: { @@ -179,7 +175,7 @@ export function defineCursorSDKInstrumentationAssertions(options: { }): void { const snapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); const testConfig = { timeout: timeoutMs }; @@ -311,11 +307,8 @@ export function defineCursorSDKInstrumentationAssertions(options: { } }); - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(summarize(events)), - snapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, snapshotPath); }); }); } diff --git a/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json b/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json deleted file mode 100644 index ea0981429..000000000 --- a/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json +++ /dev/null @@ -1,218 +0,0 @@ -[ - { - "context": {}, - "created": "", - "expected": "Paris", - "id": "", - "input": "What is the capital of France?", - "log_id": "g", - "metadata": { - "case": "basic-span", - "scenario": "deno-browser", - "testRunId": "", - "transport": "http" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": "Paris", - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "deno browser basic span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "log_id": "g", - "metadata": { - "case": "json-attachment", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "attachment": true - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "deno browser json attachment span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "phase": "parent", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "parent-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "parent" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "deno browser parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "step": "child", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "child-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "child" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "deno browser child span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-parent", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "deno browser nested parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-child", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "deno browser nested child span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-grandchild", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "deno browser nested grandchild span", - "type": "task" - }, - "span_id": "" - }, - { - "context": {}, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "current-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "deno browser current span", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-events.json b/e2e/scenarios/deno-browser/__snapshots__/span-events.json deleted file mode 100644 index 41fcf80f0..000000000 --- a/e2e/scenarios/deno-browser/__snapshots__/span-events.json +++ /dev/null @@ -1,177 +0,0 @@ -[ - { - "error": null, - "input": "What is the capital of France?", - "metadata": { - "case": "basic-span", - "scenario": "deno-browser", - "testRunId": "", - "transport": "http" - }, - "name": "deno browser basic span", - "output": "Paris", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "deno browser basic span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "metadata": { - "case": "json-attachment", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser json attachment span", - "output": { - "attachment": true - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "deno browser json attachment span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "phase": "parent", - "testRunId": "" - }, - "metadata": { - "case": "parent-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser parent span", - "output": { - "ok": true, - "phase": "parent" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "deno browser parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "step": "child", - "testRunId": "" - }, - "metadata": { - "case": "child-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser child span", - "output": { - "ok": true, - "phase": "child" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "deno browser child span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-parent", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser nested parent span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "deno browser nested parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-child", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser nested child span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "deno browser nested child span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-grandchild", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser nested grandchild span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "deno browser nested grandchild span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "current-span", - "scenario": "deno-browser", - "testRunId": "" - }, - "name": "deno browser current span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "deno browser current span", - "type": "task" - }, - "span_id": "", - "span_parents": null - } -] diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-tree.json b/e2e/scenarios/deno-browser/__snapshots__/span-tree.json new file mode 100644 index 000000000..9461d0edb --- /dev/null +++ b/e2e/scenarios/deno-browser/__snapshots__/span-tree.json @@ -0,0 +1,116 @@ +{ + "span_tree": [ + { + "name": "deno browser basic span", + "type": "task", + "children": [], + "input": "What is the capital of France?", + "output": "Paris", + "expected": "Paris", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "", + "transport": "http" + } + }, + { + "name": "deno browser json attachment span", + "type": "task", + "children": [], + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "output": { + "attachment": true + }, + "metadata": { + "case": "json-attachment", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser parent span", + "type": "task", + "children": [], + "input": { + "phase": "parent", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "parent" + }, + "metadata": { + "case": "parent-span", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser child span", + "type": "task", + "children": [], + "input": { + "step": "child", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "child" + }, + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser nested parent span", + "type": "task", + "children": [], + "metadata": { + "case": "nested-parent", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser nested child span", + "type": "task", + "children": [], + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser nested grandchild span", + "type": "task", + "children": [], + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + } + }, + { + "name": "deno browser current span", + "type": "task", + "children": [], + "metadata": { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt b/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt new file mode 100644 index 000000000..9af1c12e1 --- /dev/null +++ b/e2e/scenarios/deno-browser/__snapshots__/span-tree.txt @@ -0,0 +1,81 @@ +span_tree: +├── deno browser basic span [task] +│ input: "What is the capital of France?" +│ output: "Paris" +│ expected: "Paris" +│ metadata: { +│ "case": "basic-span", +│ "scenario": "deno-browser", +│ "testRunId": "", +│ "transport": "http" +│ } +├── deno browser json attachment span [task] +│ input: { +│ "transcript": { +│ "content_type": "application/json", +│ "filename": "conversation_transcript.json", +│ "key": "", +│ "type": "braintrust_attachment" +│ }, +│ "type": "chat_completion" +│ } +│ output: { +│ "attachment": true +│ } +│ metadata: { +│ "case": "json-attachment", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +├── deno browser parent span [task] +│ input: { +│ "phase": "parent", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "parent" +│ } +│ metadata: { +│ "case": "parent-span", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +├── deno browser child span [task] +│ input: { +│ "step": "child", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "child" +│ } +│ metadata: { +│ "case": "child-span", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +├── deno browser nested parent span [task] +│ metadata: { +│ "case": "nested-parent", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +├── deno browser nested child span [task] +│ metadata: { +│ "case": "nested-child", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +├── deno browser nested grandchild span [task] +│ metadata: { +│ "case": "nested-grandchild", +│ "scenario": "deno-browser", +│ "testRunId": "" +│ } +└── deno browser current span [task] + metadata: { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + } diff --git a/e2e/scenarios/deno-browser/scenario.test.ts b/e2e/scenarios/deno-browser/scenario.test.ts index d3752d66c..291ffbf8b 100644 --- a/e2e/scenarios/deno-browser/scenario.test.ts +++ b/e2e/scenarios/deno-browser/scenario.test.ts @@ -11,9 +11,9 @@ import { resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; -import { payloadRowsForTestRunId } from "../../helpers/wrapper-contract"; +import { summarizeRequest } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), @@ -35,7 +35,6 @@ test( async () => { await withScenarioHarness( async ({ - payloads, requestCursor, requestsAfter, runDenoScenarioDir, @@ -148,27 +147,9 @@ test( ]), ); - await matchFileSnapshot( - formatJsonFileSnapshot( - [ - basicSpan, - jsonAttachment, - parentSpan, - childSpan, - nestedParent, - nestedChild, - nestedGrandchild, - currentSpan, - ].map((event) => summarizeEvent(event!)) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - - await matchFileSnapshot( - formatJsonFileSnapshot( - payloadRowsForTestRunId(payloads(), testRunId) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + await matchSpanTreeSnapshot( + capturedEvents, + resolveFileSnapshotPath(import.meta.url, "span-tree.json"), ); await matchFileSnapshot( diff --git a/e2e/scenarios/deno-node/__snapshots__/log-payloads.json b/e2e/scenarios/deno-node/__snapshots__/log-payloads.json deleted file mode 100644 index edfe1f449..000000000 --- a/e2e/scenarios/deno-node/__snapshots__/log-payloads.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "context": { - "caller_filename": "ext:cli/40_test.js", - "caller_functionname": "innerWrapped", - "caller_lineno": 0 - }, - "created": "", - "expected": "Paris", - "id": "", - "input": "What is the capital of France?", - "log_id": "g", - "metadata": { - "case": "basic-span", - "scenario": "deno-node", - "testRunId": "", - "transport": "http" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": "Paris", - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "deno node basic span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "ext:cli/40_test.js", - "caller_functionname": "innerWrapped", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "log_id": "g", - "metadata": { - "case": "json-attachment", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "attachment": true - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "deno node json attachment span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "ext:cli/40_test.js", - "caller_functionname": "innerWrapped", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "phase": "parent", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "parent-span", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "parent" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "deno node parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", - "caller_functionname": "braintrust.traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "step": "child", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "child-span", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "child" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "deno node child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "ext:cli/40_test.js", - "caller_functionname": "async innerWrapped", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-parent", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "deno node nested parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", - "caller_functionname": "braintrust.traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-child", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "deno node nested child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", - "caller_functionname": "braintrust.traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-grandchild", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "depth": 3 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "deno node nested grandchild span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "ext:cli/40_test.js", - "caller_functionname": "async innerWrapped", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "current-span", - "scenario": "deno-node", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "observedSpanId": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "deno node current span", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/deno-node/__snapshots__/span-events.json b/e2e/scenarios/deno-node/__snapshots__/span-events.json deleted file mode 100644 index 7c1b0e158..000000000 --- a/e2e/scenarios/deno-node/__snapshots__/span-events.json +++ /dev/null @@ -1,184 +0,0 @@ -[ - { - "error": null, - "input": "What is the capital of France?", - "metadata": { - "case": "basic-span", - "scenario": "deno-node", - "testRunId": "", - "transport": "http" - }, - "name": "deno node basic span", - "output": "Paris", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "deno node basic span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "metadata": { - "case": "json-attachment", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node json attachment span", - "output": { - "attachment": true - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "deno node json attachment span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "phase": "parent", - "testRunId": "" - }, - "metadata": { - "case": "parent-span", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node parent span", - "output": { - "ok": true, - "phase": "parent" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "deno node parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "step": "child", - "testRunId": "" - }, - "metadata": { - "case": "child-span", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node child span", - "output": { - "ok": true, - "phase": "child" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "deno node child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-parent", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node nested parent span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "deno node nested parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-child", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node nested child span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "deno node nested child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-grandchild", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node nested grandchild span", - "output": { - "depth": 3 - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "deno node nested grandchild span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "current-span", - "scenario": "deno-node", - "testRunId": "" - }, - "name": "deno node current span", - "output": { - "observedSpanId": "" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "deno node current span", - "type": "task" - }, - "span_id": "", - "span_parents": null - } -] diff --git a/e2e/scenarios/deno-node/__snapshots__/span-tree.json b/e2e/scenarios/deno-node/__snapshots__/span-tree.json new file mode 100644 index 000000000..91a8a7d7e --- /dev/null +++ b/e2e/scenarios/deno-node/__snapshots__/span-tree.json @@ -0,0 +1,122 @@ +{ + "span_tree": [ + { + "name": "deno node basic span", + "type": "task", + "children": [], + "input": "What is the capital of France?", + "output": "Paris", + "expected": "Paris", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "", + "transport": "http" + } + }, + { + "name": "deno node json attachment span", + "type": "task", + "children": [], + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "output": { + "attachment": true + }, + "metadata": { + "case": "json-attachment", + "scenario": "deno-node", + "testRunId": "" + } + }, + { + "name": "deno node parent span", + "type": "task", + "children": [ + { + "name": "deno node child span", + "children": [], + "input": { + "step": "child", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "child" + }, + "metadata": { + "case": "child-span", + "scenario": "deno-node", + "testRunId": "" + } + } + ], + "input": { + "phase": "parent", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "parent" + }, + "metadata": { + "case": "parent-span", + "scenario": "deno-node", + "testRunId": "" + } + }, + { + "name": "deno node nested parent span", + "type": "task", + "children": [ + { + "name": "deno node nested child span", + "children": [ + { + "name": "deno node nested grandchild span", + "children": [], + "output": { + "depth": 3 + }, + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-node", + "testRunId": "" + } + } + ], + "metadata": { + "case": "nested-child", + "scenario": "deno-node", + "testRunId": "" + } + } + ], + "metadata": { + "case": "nested-parent", + "scenario": "deno-node", + "testRunId": "" + } + }, + { + "name": "deno node current span", + "type": "task", + "children": [], + "output": { + "observedSpanId": "" + }, + "metadata": { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/deno-node/__snapshots__/span-tree.txt b/e2e/scenarios/deno-node/__snapshots__/span-tree.txt new file mode 100644 index 000000000..9c9e4ae18 --- /dev/null +++ b/e2e/scenarios/deno-node/__snapshots__/span-tree.txt @@ -0,0 +1,87 @@ +span_tree: +├── deno node basic span [task] +│ input: "What is the capital of France?" +│ output: "Paris" +│ expected: "Paris" +│ metadata: { +│ "case": "basic-span", +│ "scenario": "deno-node", +│ "testRunId": "", +│ "transport": "http" +│ } +├── deno node json attachment span [task] +│ input: { +│ "transcript": { +│ "content_type": "application/json", +│ "filename": "conversation_transcript.json", +│ "key": "", +│ "type": "braintrust_attachment" +│ }, +│ "type": "chat_completion" +│ } +│ output: { +│ "attachment": true +│ } +│ metadata: { +│ "case": "json-attachment", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +├── deno node parent span [task] +│ input: { +│ "phase": "parent", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "parent" +│ } +│ metadata: { +│ "case": "parent-span", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +│ └── deno node child span +│ input: { +│ "step": "child", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "child" +│ } +│ metadata: { +│ "case": "child-span", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +├── deno node nested parent span [task] +│ metadata: { +│ "case": "nested-parent", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +│ └── deno node nested child span +│ metadata: { +│ "case": "nested-child", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +│ └── deno node nested grandchild span +│ output: { +│ "depth": 3 +│ } +│ metadata: { +│ "case": "nested-grandchild", +│ "scenario": "deno-node", +│ "testRunId": "" +│ } +└── deno node current span [task] + output: { + "observedSpanId": "" + } + metadata: { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + } diff --git a/e2e/scenarios/deno-node/scenario.test.ts b/e2e/scenarios/deno-node/scenario.test.ts index dd683478b..e588b6c52 100644 --- a/e2e/scenarios/deno-node/scenario.test.ts +++ b/e2e/scenarios/deno-node/scenario.test.ts @@ -11,9 +11,9 @@ import { resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; -import { payloadRowsForTestRunId } from "../../helpers/wrapper-contract"; +import { summarizeRequest } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), @@ -35,7 +35,6 @@ test( async () => { await withScenarioHarness( async ({ - payloads, requestCursor, requestsAfter, runDenoScenarioDir, @@ -158,27 +157,9 @@ test( ]), ); - await matchFileSnapshot( - formatJsonFileSnapshot( - [ - basicSpan, - jsonAttachment, - parentSpan, - childSpan, - nestedParent, - nestedChild, - nestedGrandchild, - currentSpan, - ].map((event) => summarizeEvent(event!)) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - - await matchFileSnapshot( - formatJsonFileSnapshot( - payloadRowsForTestRunId(payloads(), testRunId) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + await matchSpanTreeSnapshot( + capturedEvents, + resolveFileSnapshotPath(import.meta.url, "span-tree.json"), ); await matchFileSnapshot( diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-events.json b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-events.json deleted file mode 100644 index 45028aa48..000000000 --- a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-events.json +++ /dev/null @@ -1,239 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "genkit-instrumentation" - }, - "metric_keys": [], - "name": "genkit-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "flow" - }, - "metric_keys": [], - "name": "genkit-flow-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "instrumentationFlow", - "genkit.action_type": "flow", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.flow: instrumentationFlow", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "genkit-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "genkit.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "genkit-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "genkit.generateStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "genkit-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-embedding-001", - "provider": "genkit" - }, - "metric_keys": [], - "name": "genkit.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "function" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "operation": "model-tool" - }, - "metric_keys": [], - "name": "genkit-model-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "genkit.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "cityMarkerTool", - "genkit.action_type": "tool", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.tool: cityMarkerTool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "genkit-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "summarizeCity", - "genkit.action_type": "tool", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.tool: summarizeCity", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } -] diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.json b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.json new file mode 100644 index 000000000..cd4e8fa8a --- /dev/null +++ b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.json @@ -0,0 +1,247 @@ +{ + "span_tree": [ + { + "name": "genkit-instrumentation-root", + "type": "task", + "children": [ + { + "name": "genkit-flow-operation", + "children": [ + { + "name": "genkit.flow: instrumentationFlow", + "type": "task", + "children": [ + { + "name": "genkit-generate-operation", + "children": [ + { + "name": "genkit.generate", + "type": "llm", + "children": [], + "input": "Reply with exactly OK.", + "output": { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "OK" + } + ], + "role": "model" + }, + "output": null, + "text": "OK" + }, + "metadata": { + "finishReason": "stop", + "maxOutputTokens": 24, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 6, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "genkit-stream-operation", + "children": [ + { + "name": "genkit.generateStream", + "type": "llm", + "children": [], + "input": "Stream a short phrase.", + "output": { + "finishReason": "length", + "message": { + "content": [ + { + "text": "Here" + }, + { + "text": " are a few short phrases you could stream, depending on the context:\n\n**General" + }, + { + "text": "/Neutral:**\n\n* \"Hello there.\"\n* \"" + } + ], + "role": "model" + }, + "output": null, + "streamedText": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"", + "text": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"" + }, + "metadata": { + "finishReason": "length", + "maxOutputTokens": 32, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 32, + "prompt_tokens": 6, + "time_to_first_token": 0, + "tokens": 38 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "genkit-embed-operation", + "children": [ + { + "name": "genkit.embed", + "type": "function", + "children": [], + "input": "embed this", + "output": { + "dimensions": 3072, + "embedding_count": 1 + }, + "metadata": { + "model": "googleai/gemini-embedding-001", + "provider": "genkit" + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "genkit-tool-operation", + "children": [ + { + "name": "genkit.tool: summarizeCity", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "summary": "Vienna has precise instrumentation weather." + }, + "metadata": { + "genkit.action_key": "/tool/summarizeCity", + "genkit.action_name": "summarizeCity", + "genkit.action_type": "tool", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "genkit-model-tool-operation", + "children": [ + { + "name": "genkit.generate", + "type": "llm", + "children": [ + { + "name": "genkit.tool: cityMarkerTool", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "marker": "genkit-model-tool-called-vienna" + }, + "metadata": { + "genkit.action_key": "/tool/cityMarkerTool", + "genkit.action_name": "cityMarkerTool", + "genkit.action_type": "tool", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": "Use the cityMarkerTool tool with city Vienna before answering.", + "output": { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + ], + "role": "model" + }, + "output": null, + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + }, + "metadata": { + "finishReason": "stop", + "maxOutputTokens": 64, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 18, + "prompt_tokens": 115, + "tokens": 133 + } + } + ], + "output": { + "marker": "genkit-model-tool-called-vienna", + "toolCalled": true + }, + "metadata": { + "operation": "model-tool", + "testRunId": "" + } + } + ], + "input": "run", + "output": { + "completed": true + }, + "metadata": { + "genkit.action_key": "/flow/instrumentationFlow", + "genkit.action_name": "instrumentationFlow", + "genkit.action_type": "flow", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "flow", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "genkit-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.txt b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.txt new file mode 100644 index 000000000..c84e7d4e9 --- /dev/null +++ b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-auto.span-tree.txt @@ -0,0 +1,185 @@ +span_tree: +└── genkit-instrumentation-root [task] + metadata: { + "scenario": "genkit-instrumentation", + "testRunId": "" + } + └── genkit-flow-operation + metadata: { + "operation": "flow", + "testRunId": "" + } + └── genkit.flow: instrumentationFlow [task] + input: "run" + output: { + "completed": true + } + metadata: { + "genkit.action_key": "/flow/instrumentationFlow", + "genkit.action_name": "instrumentationFlow", + "genkit.action_type": "flow", + "provider": "genkit" + } + metrics: { + "duration": 0 + } + ├── genkit-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── genkit.generate [llm] + │ input: "Reply with exactly OK." + │ output: { + │ "finishReason": "stop", + │ "message": { + │ "content": [ + │ { + │ "text": "OK" + │ } + │ ], + │ "role": "model" + │ }, + │ "output": null, + │ "text": "OK" + │ } + │ metadata: { + │ "finishReason": "stop", + │ "maxOutputTokens": 24, + │ "model": "googleai/gemini-2.5-flash-lite", + │ "provider": "genkit", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 6, + │ "tokens": 7 + │ } + ├── genkit-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── genkit.generateStream [llm] + │ input: "Stream a short phrase." + │ output: { + │ "finishReason": "length", + │ "message": { + │ "content": [ + │ { + │ "text": "Here" + │ }, + │ { + │ "text": " are a few short phrases you could stream, depending on the context:\n\n**General" + │ }, + │ { + │ "text": "/Neutral:**\n\n* \"Hello there.\"\n* \"" + │ } + │ ], + │ "role": "model" + │ }, + │ "output": null, + │ "streamedText": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"", + │ "text": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"" + │ } + │ metadata: { + │ "finishReason": "length", + │ "maxOutputTokens": 32, + │ "model": "googleai/gemini-2.5-flash-lite", + │ "provider": "genkit", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 32, + │ "prompt_tokens": 6, + │ "time_to_first_token": 0, + │ "tokens": 38 + │ } + ├── genkit-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── genkit.embed [function] + │ input: "embed this" + │ output: { + │ "dimensions": 3072, + │ "embedding_count": 1 + │ } + │ metadata: { + │ "model": "googleai/gemini-embedding-001", + │ "provider": "genkit" + │ } + ├── genkit-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── genkit.tool: summarizeCity [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "summary": "Vienna has precise instrumentation weather." + │ } + │ metadata: { + │ "genkit.action_key": "/tool/summarizeCity", + │ "genkit.action_name": "summarizeCity", + │ "genkit.action_type": "tool", + │ "provider": "genkit" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── genkit-model-tool-operation + output: { + "marker": "genkit-model-tool-called-vienna", + "toolCalled": true + } + metadata: { + "operation": "model-tool", + "testRunId": "" + } + └── genkit.generate [llm] + input: "Use the cityMarkerTool tool with city Vienna before answering." + output: { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + ], + "role": "model" + }, + "output": null, + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + metadata: { + "finishReason": "stop", + "maxOutputTokens": 64, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + } + metrics: { + "completion_tokens": 18, + "prompt_tokens": 115, + "tokens": 133 + } + └── genkit.tool: cityMarkerTool [tool] + input: { + "city": "Vienna" + } + output: { + "marker": "genkit-model-tool-called-vienna" + } + metadata: { + "genkit.action_key": "/tool/cityMarkerTool", + "genkit.action_name": "cityMarkerTool", + "genkit.action_type": "tool", + "provider": "genkit" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-events.json b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-events.json deleted file mode 100644 index 45028aa48..000000000 --- a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-events.json +++ /dev/null @@ -1,239 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "genkit-instrumentation" - }, - "metric_keys": [], - "name": "genkit-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "flow" - }, - "metric_keys": [], - "name": "genkit-flow-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "instrumentationFlow", - "genkit.action_type": "flow", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.flow: instrumentationFlow", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "genkit-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "genkit.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "genkit-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "genkit.generateStream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "genkit-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-embedding-001", - "provider": "genkit" - }, - "metric_keys": [], - "name": "genkit.embed", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "function" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "operation": "model-tool" - }, - "metric_keys": [], - "name": "genkit-model-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "googleai/gemini-2.5-flash-lite", - "provider": "genkit" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "genkit.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "cityMarkerTool", - "genkit.action_type": "tool", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.tool: cityMarkerTool", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "genkit-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "genkit.action_name": "summarizeCity", - "genkit.action_type": "tool", - "provider": "genkit" - }, - "metric_keys": [ - "duration" - ], - "name": "genkit.tool: summarizeCity", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - } -] diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.json b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.json new file mode 100644 index 000000000..cd4e8fa8a --- /dev/null +++ b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.json @@ -0,0 +1,247 @@ +{ + "span_tree": [ + { + "name": "genkit-instrumentation-root", + "type": "task", + "children": [ + { + "name": "genkit-flow-operation", + "children": [ + { + "name": "genkit.flow: instrumentationFlow", + "type": "task", + "children": [ + { + "name": "genkit-generate-operation", + "children": [ + { + "name": "genkit.generate", + "type": "llm", + "children": [], + "input": "Reply with exactly OK.", + "output": { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "OK" + } + ], + "role": "model" + }, + "output": null, + "text": "OK" + }, + "metadata": { + "finishReason": "stop", + "maxOutputTokens": 24, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 1, + "prompt_tokens": 6, + "tokens": 7 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "genkit-stream-operation", + "children": [ + { + "name": "genkit.generateStream", + "type": "llm", + "children": [], + "input": "Stream a short phrase.", + "output": { + "finishReason": "length", + "message": { + "content": [ + { + "text": "Here" + }, + { + "text": " are a few short phrases you could stream, depending on the context:\n\n**General" + }, + { + "text": "/Neutral:**\n\n* \"Hello there.\"\n* \"" + } + ], + "role": "model" + }, + "output": null, + "streamedText": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"", + "text": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"" + }, + "metadata": { + "finishReason": "length", + "maxOutputTokens": 32, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 32, + "prompt_tokens": 6, + "time_to_first_token": 0, + "tokens": 38 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "genkit-embed-operation", + "children": [ + { + "name": "genkit.embed", + "type": "function", + "children": [], + "input": "embed this", + "output": { + "dimensions": 3072, + "embedding_count": 1 + }, + "metadata": { + "model": "googleai/gemini-embedding-001", + "provider": "genkit" + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "genkit-tool-operation", + "children": [ + { + "name": "genkit.tool: summarizeCity", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "summary": "Vienna has precise instrumentation weather." + }, + "metadata": { + "genkit.action_key": "/tool/summarizeCity", + "genkit.action_name": "summarizeCity", + "genkit.action_type": "tool", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "genkit-model-tool-operation", + "children": [ + { + "name": "genkit.generate", + "type": "llm", + "children": [ + { + "name": "genkit.tool: cityMarkerTool", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "marker": "genkit-model-tool-called-vienna" + }, + "metadata": { + "genkit.action_key": "/tool/cityMarkerTool", + "genkit.action_name": "cityMarkerTool", + "genkit.action_type": "tool", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": "Use the cityMarkerTool tool with city Vienna before answering.", + "output": { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + ], + "role": "model" + }, + "output": null, + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + }, + "metadata": { + "finishReason": "stop", + "maxOutputTokens": 64, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 18, + "prompt_tokens": 115, + "tokens": 133 + } + } + ], + "output": { + "marker": "genkit-model-tool-called-vienna", + "toolCalled": true + }, + "metadata": { + "operation": "model-tool", + "testRunId": "" + } + } + ], + "input": "run", + "output": { + "completed": true + }, + "metadata": { + "genkit.action_key": "/flow/instrumentationFlow", + "genkit.action_name": "instrumentationFlow", + "genkit.action_type": "flow", + "provider": "genkit" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "flow", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "genkit-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.txt b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.txt new file mode 100644 index 000000000..c84e7d4e9 --- /dev/null +++ b/e2e/scenarios/genkit-instrumentation/__snapshots__/genkit-v1-33-0-wrapped.span-tree.txt @@ -0,0 +1,185 @@ +span_tree: +└── genkit-instrumentation-root [task] + metadata: { + "scenario": "genkit-instrumentation", + "testRunId": "" + } + └── genkit-flow-operation + metadata: { + "operation": "flow", + "testRunId": "" + } + └── genkit.flow: instrumentationFlow [task] + input: "run" + output: { + "completed": true + } + metadata: { + "genkit.action_key": "/flow/instrumentationFlow", + "genkit.action_name": "instrumentationFlow", + "genkit.action_type": "flow", + "provider": "genkit" + } + metrics: { + "duration": 0 + } + ├── genkit-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── genkit.generate [llm] + │ input: "Reply with exactly OK." + │ output: { + │ "finishReason": "stop", + │ "message": { + │ "content": [ + │ { + │ "text": "OK" + │ } + │ ], + │ "role": "model" + │ }, + │ "output": null, + │ "text": "OK" + │ } + │ metadata: { + │ "finishReason": "stop", + │ "maxOutputTokens": 24, + │ "model": "googleai/gemini-2.5-flash-lite", + │ "provider": "genkit", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 1, + │ "prompt_tokens": 6, + │ "tokens": 7 + │ } + ├── genkit-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── genkit.generateStream [llm] + │ input: "Stream a short phrase." + │ output: { + │ "finishReason": "length", + │ "message": { + │ "content": [ + │ { + │ "text": "Here" + │ }, + │ { + │ "text": " are a few short phrases you could stream, depending on the context:\n\n**General" + │ }, + │ { + │ "text": "/Neutral:**\n\n* \"Hello there.\"\n* \"" + │ } + │ ], + │ "role": "model" + │ }, + │ "output": null, + │ "streamedText": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"", + │ "text": "Here are a few short phrases you could stream, depending on the context:\n\n**General/Neutral:**\n\n* \"Hello there.\"\n* \"" + │ } + │ metadata: { + │ "finishReason": "length", + │ "maxOutputTokens": 32, + │ "model": "googleai/gemini-2.5-flash-lite", + │ "provider": "genkit", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 32, + │ "prompt_tokens": 6, + │ "time_to_first_token": 0, + │ "tokens": 38 + │ } + ├── genkit-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── genkit.embed [function] + │ input: "embed this" + │ output: { + │ "dimensions": 3072, + │ "embedding_count": 1 + │ } + │ metadata: { + │ "model": "googleai/gemini-embedding-001", + │ "provider": "genkit" + │ } + ├── genkit-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── genkit.tool: summarizeCity [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "summary": "Vienna has precise instrumentation weather." + │ } + │ metadata: { + │ "genkit.action_key": "/tool/summarizeCity", + │ "genkit.action_name": "summarizeCity", + │ "genkit.action_type": "tool", + │ "provider": "genkit" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── genkit-model-tool-operation + output: { + "marker": "genkit-model-tool-called-vienna", + "toolCalled": true + } + metadata: { + "operation": "model-tool", + "testRunId": "" + } + └── genkit.generate [llm] + input: "Use the cityMarkerTool tool with city Vienna before answering." + output: { + "finishReason": "stop", + "message": { + "content": [ + { + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + ], + "role": "model" + }, + "output": null, + "text": "The canonical marker for Vienna is genkit-model-tool-called-vienna." + } + metadata: { + "finishReason": "stop", + "maxOutputTokens": 64, + "model": "googleai/gemini-2.5-flash-lite", + "provider": "genkit", + "temperature": 0 + } + metrics: { + "completion_tokens": 18, + "prompt_tokens": 115, + "tokens": 133 + } + └── genkit.tool: cityMarkerTool [tool] + input: { + "city": "Vienna" + } + output: { + "marker": "genkit-model-tool-called-vienna" + } + metadata: { + "genkit.action_key": "/tool/cityMarkerTool", + "genkit.action_name": "cityMarkerTool", + "genkit.action_type": "tool", + "provider": "genkit" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/genkit-instrumentation/assertions.ts b/e2e/scenarios/genkit-instrumentation/assertions.ts index eeb05bf7d..1b937892f 100644 --- a/e2e/scenarios/genkit-instrumentation/assertions.ts +++ b/e2e/scenarios/genkit-instrumentation/assertions.ts @@ -1,17 +1,15 @@ import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { MODEL_TOOL_MARKER, ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; type RunGenkitScenario = (harness: { @@ -49,7 +47,7 @@ function expectChildOf( function buildSpanSummary( events: CapturedLogEvent[], supportsActionSpans: boolean, -): Json { +): SpanTreeEntry[] { const flowOperation = findLatestSpan(events, "genkit-flow-operation"); const flowSpan = supportsActionSpans ? findGenkitSpan( @@ -105,17 +103,7 @@ function buildSpanSummary( ); } - return summary.map((event) => - summarizeWrapperContract(event!, [ - "genkit.action_name", - "genkit.action_type", - "genkit.run_name", - "model", - "operation", - "provider", - "scenario", - ]), - ) as Json; + return summary.map((event) => ({ event: event! })); } export function defineGenkitInstrumentationAssertions(options: { @@ -128,7 +116,7 @@ export function defineGenkitInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -303,13 +291,8 @@ export function defineGenkitInstrumentationAssertions(options: { }, ); - test("matches span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildSpanSummary(events, options.supportsActionSpans), - ), - spanSnapshotPath, - ); + test("matches span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-events.json b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-events.json deleted file mode 100644 index 4f6898690..000000000 --- a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-events.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "basic": { - "llm": { - "has_input": false, - "has_output": false, - "metadata": { - "model": "gpt-4.1" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "prompt_tokens", - "reasoning_tokens", - "tokens" - ], - "name": "github.copilot.llm", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "github-copilot-basic-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "session": { - "has_input": false, - "has_output": false, - "metadata": { - "github_copilot.end_reason": "complete", - "github_copilot.model": "gpt-4.1", - "github_copilot.provider_type": "openai" - }, - "metric_keys": [], - "name": "Copilot Session", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "turn": { - "has_input": true, - "has_output": true, - "metadata": null, - "metric_keys": [], - "name": "Copilot Turn", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "github-copilot-instrumentation" - }, - "metric_keys": [], - "name": "github-copilot-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "tool": { - "llm": { - "has_input": false, - "has_output": false, - "metadata": { - "model": "gpt-4.1" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "prompt_tokens", - "reasoning_tokens", - "tokens" - ], - "name": "github.copilot.llm", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "github-copilot-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "session": { - "has_input": false, - "has_output": false, - "metadata": { - "github_copilot.end_reason": "complete", - "github_copilot.model": "gpt-4.1", - "github_copilot.provider_type": "openai" - }, - "metric_keys": [], - "name": "Copilot Session", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "get_weather" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - "turn": { - "has_input": false, - "has_output": true, - "metadata": null, - "metric_keys": [], - "name": "Copilot Turn", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - } -} diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.json b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.json new file mode 100644 index 000000000..5b6f4ef6a --- /dev/null +++ b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.json @@ -0,0 +1,166 @@ +{ + "span_tree": [ + { + "name": "github-copilot-instrumentation-root", + "type": "task", + "children": [ + { + "name": "github-copilot-basic-operation", + "children": [ + { + "name": "Copilot Session", + "type": "task", + "children": [ + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7nGaoedcSfrmf6rVMJ9F1bE71S", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "user", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 6144, + "prompt_tokens": 18585, + "reasoning_tokens": 0, + "tokens": 18587 + } + } + ], + "input": "Reply with exactly: OK", + "output": "OK" + } + ], + "metadata": { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 2, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 12441, + "github_copilot.total_output_tokens": 2 + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "github-copilot-tool-operation", + "children": [ + { + "name": "Copilot Session", + "type": "task", + "children": [ + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7qQSE3SNFqkisyWCQwNUeQlzs4", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "user", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 14, + "prompt_cached_tokens": 6144, + "prompt_tokens": 18606, + "reasoning_tokens": 0, + "tokens": 18620 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "city": "Tokyo" + }, + "output": { + "content": "The weather in Tokyo is 72°F and sunny.", + "detailedContent": "The weather in Tokyo is 72°F and sunny." + }, + "metadata": { + "gen_ai.tool.call.id": "call_EtVZ0wzZObwD18ZoOOzxhR6U", + "gen_ai.tool.name": "get_weather" + } + } + ], + "input": "What is the weather in Tokyo? Use the get_weather tool and report the result.", + "output": "" + }, + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7rX2REjanlVkzA24AuyMy1zaQZ", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "agent", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 13, + "prompt_cached_tokens": 12416, + "prompt_tokens": 24911, + "reasoning_tokens": 0, + "tokens": 24924 + } + } + ], + "output": "The weather in Tokyo is currently 72°F and sunny." + } + ], + "metadata": { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 4, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 24957, + "github_copilot.total_output_tokens": 27 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "github-copilot-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.txt b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.txt new file mode 100644 index 000000000..37f50893a --- /dev/null +++ b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-auto.span-tree.txt @@ -0,0 +1,109 @@ +span_tree: +└── github-copilot-instrumentation-root [task] + metadata: { + "scenario": "github-copilot-instrumentation", + "testRunId": "" + } + ├── github-copilot-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Copilot Session [task] + │ metadata: { + │ "github_copilot.context_window.current": 0, + │ "github_copilot.context_window.limit": 128000, + │ "github_copilot.context_window.messages": 2, + │ "github_copilot.end_reason": "complete", + │ "github_copilot.model": "gpt-4.1", + │ "github_copilot.provider_type": "openai", + │ "github_copilot.total_input_tokens": 12441, + │ "github_copilot.total_output_tokens": 2 + │ } + │ └── Copilot Turn [task] + │ input: "Reply with exactly: OK" + │ output: "OK" + │ └── github.copilot.llm [llm] + │ metadata: { + │ "github_copilot.api_call_id": "chatcmpl-Dfl7nGaoedcSfrmf6rVMJ9F1bE71S", + │ "github_copilot.cost": 0, + │ "github_copilot.duration_ms": 0, + │ "github_copilot.initiator": "user", + │ "github_copilot.quota_snapshots": {}, + │ "model": "gpt-4.1" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 6144, + │ "prompt_tokens": 18585, + │ "reasoning_tokens": 0, + │ "tokens": 18587 + │ } + └── github-copilot-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── Copilot Session [task] + metadata: { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 4, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 24957, + "github_copilot.total_output_tokens": 27 + } + ├── Copilot Turn [task] + │ input: "What is the weather in Tokyo? Use the get_weather tool and report the result." + │ output: "" + │ ├── github.copilot.llm [llm] + │ │ metadata: { + │ │ "github_copilot.api_call_id": "chatcmpl-Dfl7qQSE3SNFqkisyWCQwNUeQlzs4", + │ │ "github_copilot.cost": 0, + │ │ "github_copilot.duration_ms": 0, + │ │ "github_copilot.initiator": "user", + │ │ "github_copilot.quota_snapshots": {}, + │ │ "model": "gpt-4.1" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 14, + │ │ "prompt_cached_tokens": 6144, + │ │ "prompt_tokens": 18606, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 18620 + │ │ } + │ └── tool: get_weather [tool] + │ input: { + │ "city": "Tokyo" + │ } + │ output: { + │ "content": "The weather in Tokyo is 72°F and sunny.", + │ "detailedContent": "The weather in Tokyo is 72°F and sunny." + │ } + │ metadata: { + │ "gen_ai.tool.call.id": "call_EtVZ0wzZObwD18ZoOOzxhR6U", + │ "gen_ai.tool.name": "get_weather" + │ } + └── Copilot Turn [task] + output: "The weather in Tokyo is currently 72°F and sunny." + └── github.copilot.llm [llm] + metadata: { + "github_copilot.api_call_id": "chatcmpl-Dfl7rX2REjanlVkzA24AuyMy1zaQZ", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "agent", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 13, + "prompt_cached_tokens": 12416, + "prompt_tokens": 24911, + "reasoning_tokens": 0, + "tokens": 24924 + } diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-events.json b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-events.json deleted file mode 100644 index 4f6898690..000000000 --- a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-events.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "basic": { - "llm": { - "has_input": false, - "has_output": false, - "metadata": { - "model": "gpt-4.1" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "prompt_tokens", - "reasoning_tokens", - "tokens" - ], - "name": "github.copilot.llm", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "github-copilot-basic-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "session": { - "has_input": false, - "has_output": false, - "metadata": { - "github_copilot.end_reason": "complete", - "github_copilot.model": "gpt-4.1", - "github_copilot.provider_type": "openai" - }, - "metric_keys": [], - "name": "Copilot Session", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "turn": { - "has_input": true, - "has_output": true, - "metadata": null, - "metric_keys": [], - "name": "Copilot Turn", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "github-copilot-instrumentation" - }, - "metric_keys": [], - "name": "github-copilot-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "tool": { - "llm": { - "has_input": false, - "has_output": false, - "metadata": { - "model": "gpt-4.1" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "prompt_tokens", - "reasoning_tokens", - "tokens" - ], - "name": "github.copilot.llm", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - "operation": { - "has_input": false, - "has_output": false, - "metadata": null, - "metric_keys": [], - "name": "github-copilot-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "session": { - "has_input": false, - "has_output": false, - "metadata": { - "github_copilot.end_reason": "complete", - "github_copilot.model": "gpt-4.1", - "github_copilot.provider_type": "openai" - }, - "metric_keys": [], - "name": "Copilot Session", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - "tool": { - "has_input": true, - "has_output": true, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "get_weather" - }, - "metric_keys": [], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - "turn": { - "has_input": false, - "has_output": true, - "metadata": null, - "metric_keys": [], - "name": "Copilot Turn", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - } -} diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.json b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.json new file mode 100644 index 000000000..21f18e6e1 --- /dev/null +++ b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.json @@ -0,0 +1,165 @@ +{ + "span_tree": [ + { + "name": "github-copilot-instrumentation-root", + "type": "task", + "children": [ + { + "name": "github-copilot-basic-operation", + "children": [ + { + "name": "Copilot Session", + "type": "task", + "children": [ + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7WGPHDVLLpblipA24xvJneX5M3", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "user", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_tokens": 12438, + "reasoning_tokens": 0, + "tokens": 12440 + } + } + ], + "input": "Reply with exactly: OK", + "output": "OK" + } + ], + "metadata": { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 2, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 12438, + "github_copilot.total_output_tokens": 2 + } + } + ], + "metadata": { + "operation": "basic", + "testRunId": "" + } + }, + { + "name": "github-copilot-tool-operation", + "children": [ + { + "name": "Copilot Session", + "type": "task", + "children": [ + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7ZFqD3Hyt7vYsJ5lyihhLcfsHh", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "user", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 14, + "prompt_cached_tokens": 6144, + "prompt_tokens": 18597, + "reasoning_tokens": 0, + "tokens": 18611 + } + }, + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "city": "Tokyo" + }, + "output": { + "content": "The weather in Tokyo is 72°F and sunny.", + "detailedContent": "The weather in Tokyo is 72°F and sunny." + }, + "metadata": { + "gen_ai.tool.call.id": "call_iCtypQVOy9ZxGGrQf3tQ76rH", + "gen_ai.tool.name": "get_weather" + } + } + ], + "input": "What is the weather in Tokyo? Use the get_weather tool and report the result.", + "output": "" + }, + { + "name": "Copilot Turn", + "type": "task", + "children": [ + { + "name": "github.copilot.llm", + "type": "llm", + "children": [], + "metadata": { + "github_copilot.api_call_id": "chatcmpl-Dfl7aHH1Xd1u3v2rFNY3Fy9OKYcIh", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "agent", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 13, + "prompt_cached_tokens": 12416, + "prompt_tokens": 24902, + "reasoning_tokens": 0, + "tokens": 24915 + } + } + ], + "output": "The weather in Tokyo is currently 72°F and sunny." + } + ], + "metadata": { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 4, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 24939, + "github_copilot.total_output_tokens": 27 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "github-copilot-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.txt b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.txt new file mode 100644 index 000000000..4c23bd256 --- /dev/null +++ b/e2e/scenarios/github-copilot-instrumentation/__snapshots__/github-copilot-v0-wrapped.span-tree.txt @@ -0,0 +1,108 @@ +span_tree: +└── github-copilot-instrumentation-root [task] + metadata: { + "scenario": "github-copilot-instrumentation", + "testRunId": "" + } + ├── github-copilot-basic-operation + │ metadata: { + │ "operation": "basic", + │ "testRunId": "" + │ } + │ └── Copilot Session [task] + │ metadata: { + │ "github_copilot.context_window.current": 0, + │ "github_copilot.context_window.limit": 128000, + │ "github_copilot.context_window.messages": 2, + │ "github_copilot.end_reason": "complete", + │ "github_copilot.model": "gpt-4.1", + │ "github_copilot.provider_type": "openai", + │ "github_copilot.total_input_tokens": 12438, + │ "github_copilot.total_output_tokens": 2 + │ } + │ └── Copilot Turn [task] + │ input: "Reply with exactly: OK" + │ output: "OK" + │ └── github.copilot.llm [llm] + │ metadata: { + │ "github_copilot.api_call_id": "chatcmpl-Dfl7WGPHDVLLpblipA24xvJneX5M3", + │ "github_copilot.cost": 0, + │ "github_copilot.duration_ms": 0, + │ "github_copilot.initiator": "user", + │ "github_copilot.quota_snapshots": {}, + │ "model": "gpt-4.1" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_tokens": 12438, + │ "reasoning_tokens": 0, + │ "tokens": 12440 + │ } + └── github-copilot-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── Copilot Session [task] + metadata: { + "github_copilot.context_window.current": 0, + "github_copilot.context_window.limit": 128000, + "github_copilot.context_window.messages": 4, + "github_copilot.end_reason": "complete", + "github_copilot.model": "gpt-4.1", + "github_copilot.provider_type": "openai", + "github_copilot.total_input_tokens": 24939, + "github_copilot.total_output_tokens": 27 + } + ├── Copilot Turn [task] + │ input: "What is the weather in Tokyo? Use the get_weather tool and report the result." + │ output: "" + │ ├── github.copilot.llm [llm] + │ │ metadata: { + │ │ "github_copilot.api_call_id": "chatcmpl-Dfl7ZFqD3Hyt7vYsJ5lyihhLcfsHh", + │ │ "github_copilot.cost": 0, + │ │ "github_copilot.duration_ms": 0, + │ │ "github_copilot.initiator": "user", + │ │ "github_copilot.quota_snapshots": {}, + │ │ "model": "gpt-4.1" + │ │ } + │ │ metrics: { + │ │ "completion_reasoning_tokens": 0, + │ │ "completion_tokens": 14, + │ │ "prompt_cached_tokens": 6144, + │ │ "prompt_tokens": 18597, + │ │ "reasoning_tokens": 0, + │ │ "tokens": 18611 + │ │ } + │ └── tool: get_weather [tool] + │ input: { + │ "city": "Tokyo" + │ } + │ output: { + │ "content": "The weather in Tokyo is 72°F and sunny.", + │ "detailedContent": "The weather in Tokyo is 72°F and sunny." + │ } + │ metadata: { + │ "gen_ai.tool.call.id": "call_iCtypQVOy9ZxGGrQf3tQ76rH", + │ "gen_ai.tool.name": "get_weather" + │ } + └── Copilot Turn [task] + output: "The weather in Tokyo is currently 72°F and sunny." + └── github.copilot.llm [llm] + metadata: { + "github_copilot.api_call_id": "chatcmpl-Dfl7aHH1Xd1u3v2rFNY3Fy9OKYcIh", + "github_copilot.cost": 0, + "github_copilot.duration_ms": 0, + "github_copilot.initiator": "agent", + "github_copilot.quota_snapshots": {}, + "model": "gpt-4.1" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 13, + "prompt_cached_tokens": 12416, + "prompt_tokens": 24902, + "reasoning_tokens": 0, + "tokens": 24915 + } diff --git a/e2e/scenarios/github-copilot-instrumentation/assertions.ts b/e2e/scenarios/github-copilot-instrumentation/assertions.ts index a77d43b8f..14812767d 100644 --- a/e2e/scenarios/github-copilot-instrumentation/assertions.ts +++ b/e2e/scenarios/github-copilot-instrumentation/assertions.ts @@ -1,17 +1,17 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, + type SpanTreeFields, +} from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; type RunCopilotScenario = (harness: { @@ -42,32 +42,43 @@ const SNAPSHOT_METADATA_KEYS = [ "github_copilot.agent_name", ] as const; -function summarizeSpan(event: CapturedLogEvent | undefined): Json { - if (!event) { - return null; - } - - const summary = summarizeWrapperContract(event, [ - ...SNAPSHOT_METADATA_KEYS, - ]) as Record; +function snapshotFields(event: CapturedLogEvent): SpanTreeFields { + const fields = spanTreeFields(event); + const metadata = + fields.metadata && + typeof fields.metadata === "object" && + !Array.isArray(fields.metadata) + ? Object.fromEntries( + Object.entries(fields.metadata).filter(([key]) => + SNAPSHOT_METADATA_KEYS.includes( + key as (typeof SNAPSHOT_METADATA_KEYS)[number], + ), + ), + ) + : undefined; - // Normalize non-deterministic IDs in metadata - if (summary.metadata && typeof summary.metadata === "object") { - const metadata = summary.metadata as Record; - if (typeof metadata["gen_ai.tool.call.id"] === "string") { - metadata["gen_ai.tool.call.id"] = ""; - } + if (metadata && typeof metadata["gen_ai.tool.call.id"] === "string") { + metadata["gen_ai.tool.call.id"] = ""; } - if (Array.isArray(summary.metric_keys)) { - summary.metric_keys = summary.metric_keys.filter( - (key) => key !== "prompt_cached_tokens", - ); + + const metrics = + fields.metrics && + typeof fields.metrics === "object" && + !Array.isArray(fields.metrics) + ? ({ ...fields.metrics } as Record) + : undefined; + if (metrics) { + delete metrics.prompt_cached_tokens; } - return summary; + return { + ...fields, + metadata, + metrics, + }; } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { const root = findLatestSpan(events, ROOT_NAME); const basicOperation = findLatestSpan( events, @@ -125,22 +136,21 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { ) .at(-1); - return normalizeForSnapshot({ - root: summarizeSpan(root), - basic: { - operation: summarizeSpan(basicOperation), - session: summarizeSpan(basicSession), - turn: summarizeSpan(basicTurn), - llm: summarizeSpan(basicLlm), - }, - tool: { - operation: summarizeSpan(toolOperation), - session: summarizeSpan(toolSession), - turn: summarizeSpan(toolTurn), - llm: summarizeSpan(toolLlm), - tool: summarizeSpan(toolSpan), - }, - } as Json); + return [ + root, + basicOperation, + basicSession, + basicTurn, + basicLlm, + toolOperation, + toolSession, + toolTurn, + toolLlm, + toolSpan, + ].map((event) => ({ + event: event!, + fields: snapshotFields(event!), + })); } export function defineGitHubCopilotInstrumentationAssertions(options: { @@ -152,7 +162,7 @@ export function defineGitHubCopilotInstrumentationAssertions(options: { }): void { const snapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs }; @@ -253,11 +263,8 @@ export function defineGitHubCopilotInstrumentationAssertions(options: { expect(toolSpan?.span.type).toBe("tool"); }); - test("matches the span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events)), - snapshotPath, - ); + test("matches the span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, snapshotPath); }); }); } diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.json new file mode 100644 index 000000000..1e1d4eed4 --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.json @@ -0,0 +1,263 @@ +{ + "span_tree": [ + { + "name": "google-adk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "adk-simple-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: weather_agent", + "type": "task", + "children": [ + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "location": "Paris, France" + }, + "output": { + "condition": "sunny", + "location": "Paris, France", + "temperature": 72 + }, + "metadata": { + "google_adk.tool_call_id": "direct-weather-tool", + "google_adk.tool_name": "get_weather", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "text": "What is the weather in Paris, France?" + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The weather in Paris, France is currently 15°C and cloudy." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 16, + "promptTokenCount": 37, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 37 + } + ], + "serviceTier": "standard", + "totalTokenCount": 53 + } + }, + "metadata": { + "model": "gemini-2.5-flash-lite", + "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "weather_agent", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "What is the weather in Paris, France?", + "role": "user" + } + ] + }, + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-1", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "metadata": { + "operation": "simple-run", + "testRunId": "" + } + }, + { + "name": "adk-sequential-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: sequential_workflow", + "type": "task", + "children": [ + { + "name": "Agent: greeter", + "type": "task", + "children": [], + "output": { + "author": "greeter", + "content": "Hello.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "greeter", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "Agent: farewell", + "type": "task", + "children": [], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + }, + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "sequential-run", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.txt b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.txt new file mode 100644 index 000000000..0177fca3e --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-auto-hook.span-tree.txt @@ -0,0 +1,211 @@ +span_tree: +└── google-adk-instrumentation-root [task] + metadata: { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + ├── adk-simple-run-operation + │ metadata: { + │ "operation": "simple-run", + │ "testRunId": "" + │ } + │ └── Google ADK Runner [task] + │ input: { + │ "messages": [ + │ { + │ "content": "What is the weather in Paris, France?", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.session_id": "test-session-1", + │ "google_adk.user_id": "test-user", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + │ └── Agent: weather_agent [task] + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "weather_agent", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + │ ├── tool: get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperature": 72 + │ │ } + │ │ metadata: { + │ │ "google_adk.tool_call_id": "direct-weather-tool", + │ │ "google_adk.tool_name": "get_weather", + │ │ "provider": "google-adk" + │ │ } + │ │ metrics: { + │ │ "duration": 0 + │ │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "text": "What is the weather in Paris, France?" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The weather in Paris, France is currently 15°C and cloudy." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 16, + │ "promptTokenCount": 37, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 37 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 53 + │ } + │ } + │ metadata: { + │ "model": "gemini-2.5-flash-lite", + │ "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + └── adk-sequential-run-operation + metadata: { + "operation": "sequential-run", + "testRunId": "" + } + └── Google ADK Runner [task] + input: { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + } + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + └── Agent: sequential_workflow [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + ├── Agent: greeter [task] + │ output: { + │ "author": "greeter", + │ "content": "Hello.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "greeter", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── Agent: farewell [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.json new file mode 100644 index 000000000..82b9c464e --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.json @@ -0,0 +1,183 @@ +{ + "span_tree": [ + { + "name": "google-adk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "adk-simple-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: weather_agent", + "type": "task", + "children": [ + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "location": "Paris, France" + }, + "output": { + "condition": "sunny", + "location": "Paris, France", + "temperature": 72 + }, + "metadata": { + "google_adk.tool_call_id": "direct-weather-tool", + "google_adk.tool_name": "get_weather", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "weather_agent", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "What is the weather in Paris, France?", + "role": "user" + } + ] + }, + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-1", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "metadata": { + "operation": "simple-run", + "testRunId": "" + } + }, + { + "name": "adk-sequential-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: sequential_workflow", + "type": "task", + "children": [ + { + "name": "Agent: greeter", + "type": "task", + "children": [], + "output": { + "author": "greeter", + "content": "Hello.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "greeter", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "Agent: farewell", + "type": "task", + "children": [], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + }, + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "sequential-run", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.txt b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.txt new file mode 100644 index 000000000..66f2073bf --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061-wrapped.span-tree.txt @@ -0,0 +1,135 @@ +span_tree: +└── google-adk-instrumentation-root [task] + metadata: { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + ├── adk-simple-run-operation + │ metadata: { + │ "operation": "simple-run", + │ "testRunId": "" + │ } + │ └── Google ADK Runner [task] + │ input: { + │ "messages": [ + │ { + │ "content": "What is the weather in Paris, France?", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.session_id": "test-session-1", + │ "google_adk.user_id": "test-user", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + │ └── Agent: weather_agent [task] + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "weather_agent", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + │ └── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperature": 72 + │ } + │ metadata: { + │ "google_adk.tool_call_id": "direct-weather-tool", + │ "google_adk.tool_name": "get_weather", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── adk-sequential-run-operation + metadata: { + "operation": "sequential-run", + "testRunId": "" + } + └── Google ADK Runner [task] + input: { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + } + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + └── Agent: sequential_workflow [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + ├── Agent: greeter [task] + │ output: { + │ "author": "greeter", + │ "content": "Hello.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "greeter", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── Agent: farewell [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.log-payloads.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.log-payloads.json deleted file mode 100644 index 681305015..000000000 --- a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.log-payloads.json +++ /dev/null @@ -1,156 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-adk-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-adk-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "simple-run" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "adk-simple-run-operation", - "type": null - }, - { - "metadata": { - "operation": "sequential-run" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "adk-sequential-run-operation", - "type": null - }, - { - "input": { - "messages": [ - { - "content": "What is the weather in Paris, France?", - "role": "user" - } - ] - }, - "metadata": { - "google_adk.session_id": "test-session-1", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Google ADK Runner", - "type": "task" - }, - { - "input": { - "messages": [ - { - "content": "Hello.", - "role": "user" - } - ] - }, - "metadata": { - "google_adk.session_id": "test-session-sequential", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Google ADK Runner", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "weather_agent", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: weather_agent", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "sequential_workflow", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: sequential_workflow", - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "google_adk.tool_call_id": "direct-weather-tool", - "google_adk.tool_name": "get_weather", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": { - "condition": "sunny", - "location": "Paris, France", - "temperature": 72 - }, - "type": "tool" - }, - { - "metadata": { - "google_adk.agent_name": "greeter", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: greeter", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "farewell", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: farewell", - "type": "task" - } -] diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.span-events.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.span-events.json deleted file mode 100644 index 656a606a1..000000000 --- a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v061.span-events.json +++ /dev/null @@ -1,171 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-adk-instrumentation" - }, - "metric_keys": [], - "name": "google-adk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "simple-run" - }, - "metric_keys": [], - "name": "adk-simple-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "sequential-run" - }, - "metric_keys": [], - "name": "adk-sequential-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "metadata": { - "google_adk.session_id": "test-session-1", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Google ADK Runner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "metadata": { - "google_adk.session_id": "test-session-sequential", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Google ADK Runner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "weather_agent", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: weather_agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "sequential_workflow", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: sequential_workflow", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "google_adk.tool_call_id": "direct-weather-tool", - "google_adk.tool_name": "get_weather", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "greeter", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: greeter", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "farewell", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: farewell", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } -] diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.json new file mode 100644 index 000000000..1e1d4eed4 --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.json @@ -0,0 +1,263 @@ +{ + "span_tree": [ + { + "name": "google-adk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "adk-simple-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: weather_agent", + "type": "task", + "children": [ + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "location": "Paris, France" + }, + "output": { + "condition": "sunny", + "location": "Paris, France", + "temperature": 72 + }, + "metadata": { + "google_adk.tool_call_id": "direct-weather-tool", + "google_adk.tool_name": "get_weather", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "text": "What is the weather in Paris, France?" + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The weather in Paris, France is currently 15°C and cloudy." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 16, + "promptTokenCount": 37, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 37 + } + ], + "serviceTier": "standard", + "totalTokenCount": 53 + } + }, + "metadata": { + "model": "gemini-2.5-flash-lite", + "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "weather_agent", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "What is the weather in Paris, France?", + "role": "user" + } + ] + }, + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-1", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "metadata": { + "operation": "simple-run", + "testRunId": "" + } + }, + { + "name": "adk-sequential-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: sequential_workflow", + "type": "task", + "children": [ + { + "name": "Agent: greeter", + "type": "task", + "children": [], + "output": { + "author": "greeter", + "content": "Hello.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "greeter", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "Agent: farewell", + "type": "task", + "children": [], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + }, + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "sequential-run", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.txt b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.txt new file mode 100644 index 000000000..0177fca3e --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-auto-hook.span-tree.txt @@ -0,0 +1,211 @@ +span_tree: +└── google-adk-instrumentation-root [task] + metadata: { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + ├── adk-simple-run-operation + │ metadata: { + │ "operation": "simple-run", + │ "testRunId": "" + │ } + │ └── Google ADK Runner [task] + │ input: { + │ "messages": [ + │ { + │ "content": "What is the weather in Paris, France?", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.session_id": "test-session-1", + │ "google_adk.user_id": "test-user", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + │ └── Agent: weather_agent [task] + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "weather_agent", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + │ ├── tool: get_weather [tool] + │ │ input: { + │ │ "location": "Paris, France" + │ │ } + │ │ output: { + │ │ "condition": "sunny", + │ │ "location": "Paris, France", + │ │ "temperature": 72 + │ │ } + │ │ metadata: { + │ │ "google_adk.tool_call_id": "direct-weather-tool", + │ │ "google_adk.tool_name": "get_weather", + │ │ "provider": "google-adk" + │ │ } + │ │ metrics: { + │ │ "duration": 0 + │ │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "text": "What is the weather in Paris, France?" + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The weather in Paris, France is currently 15°C and cloudy." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 16, + │ "promptTokenCount": 37, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 37 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 53 + │ } + │ } + │ metadata: { + │ "model": "gemini-2.5-flash-lite", + │ "systemInstruction": "You are an agent. Your internal name is \"weather_agent\".\n\nAnswer the user's question in one short sentence.", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + └── adk-sequential-run-operation + metadata: { + "operation": "sequential-run", + "testRunId": "" + } + └── Google ADK Runner [task] + input: { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + } + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + └── Agent: sequential_workflow [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + ├── Agent: greeter [task] + │ output: { + │ "author": "greeter", + │ "content": "Hello.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "greeter", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── Agent: farewell [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.json new file mode 100644 index 000000000..82b9c464e --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.json @@ -0,0 +1,183 @@ +{ + "span_tree": [ + { + "name": "google-adk-instrumentation-root", + "type": "task", + "children": [ + { + "name": "adk-simple-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: weather_agent", + "type": "task", + "children": [ + { + "name": "tool: get_weather", + "type": "tool", + "children": [], + "input": { + "location": "Paris, France" + }, + "output": { + "condition": "sunny", + "location": "Paris, France", + "temperature": 72 + }, + "metadata": { + "google_adk.tool_call_id": "direct-weather-tool", + "google_adk.tool_name": "get_weather", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "weather_agent", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "What is the weather in Paris, France?", + "role": "user" + } + ] + }, + "output": { + "author": "weather_agent", + "content": "The weather in Paris, France is currently 15°C and cloudy.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-1", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "completion_tokens": 16, + "duration": 0, + "prompt_tokens": 37, + "tokens": 53 + } + } + ], + "metadata": { + "operation": "simple-run", + "testRunId": "" + } + }, + { + "name": "adk-sequential-run-operation", + "children": [ + { + "name": "Google ADK Runner", + "type": "task", + "children": [ + { + "name": "Agent: sequential_workflow", + "type": "task", + "children": [ + { + "name": "Agent: greeter", + "type": "task", + "children": [], + "output": { + "author": "greeter", + "content": "Hello.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "greeter", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + }, + { + "name": "Agent: farewell", + "type": "task", + "children": [], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "input": { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + }, + "output": { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + }, + "metadata": { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "sequential-run", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.txt b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.txt new file mode 100644 index 000000000..66f2073bf --- /dev/null +++ b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000-wrapped.span-tree.txt @@ -0,0 +1,135 @@ +span_tree: +└── google-adk-instrumentation-root [task] + metadata: { + "scenario": "google-adk-instrumentation", + "testRunId": "" + } + ├── adk-simple-run-operation + │ metadata: { + │ "operation": "simple-run", + │ "testRunId": "" + │ } + │ └── Google ADK Runner [task] + │ input: { + │ "messages": [ + │ { + │ "content": "What is the weather in Paris, France?", + │ "role": "user" + │ } + │ ] + │ } + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.session_id": "test-session-1", + │ "google_adk.user_id": "test-user", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "duration": 0, + │ "prompt_tokens": 37, + │ "tokens": 53 + │ } + │ └── Agent: weather_agent [task] + │ output: { + │ "author": "weather_agent", + │ "content": "The weather in Paris, France is currently 15°C and cloudy.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "weather_agent", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + │ └── tool: get_weather [tool] + │ input: { + │ "location": "Paris, France" + │ } + │ output: { + │ "condition": "sunny", + │ "location": "Paris, France", + │ "temperature": 72 + │ } + │ metadata: { + │ "google_adk.tool_call_id": "direct-weather-tool", + │ "google_adk.tool_name": "get_weather", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── adk-sequential-run-operation + metadata: { + "operation": "sequential-run", + "testRunId": "" + } + └── Google ADK Runner [task] + input: { + "messages": [ + { + "content": "Hello.", + "role": "user" + } + ] + } + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.session_id": "test-session-sequential", + "google_adk.user_id": "test-user", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + └── Agent: sequential_workflow [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "sequential_workflow", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } + ├── Agent: greeter [task] + │ output: { + │ "author": "greeter", + │ "content": "Hello.", + │ "role": "model" + │ } + │ metadata: { + │ "google_adk.agent_name": "greeter", + │ "model": "gemini-2.5-flash-lite", + │ "provider": "google-adk" + │ } + │ metrics: { + │ "duration": 0 + │ } + └── Agent: farewell [task] + output: { + "author": "farewell", + "content": "Goodbye.", + "role": "model" + } + metadata: { + "google_adk.agent_name": "farewell", + "model": "gemini-2.5-flash-lite", + "provider": "google-adk" + } + metrics: { + "duration": 0 + } diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.log-payloads.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.log-payloads.json deleted file mode 100644 index 681305015..000000000 --- a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.log-payloads.json +++ /dev/null @@ -1,156 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-adk-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-adk-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "simple-run" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "adk-simple-run-operation", - "type": null - }, - { - "metadata": { - "operation": "sequential-run" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "adk-sequential-run-operation", - "type": null - }, - { - "input": { - "messages": [ - { - "content": "What is the weather in Paris, France?", - "role": "user" - } - ] - }, - "metadata": { - "google_adk.session_id": "test-session-1", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Google ADK Runner", - "type": "task" - }, - { - "input": { - "messages": [ - { - "content": "Hello.", - "role": "user" - } - ] - }, - "metadata": { - "google_adk.session_id": "test-session-sequential", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Google ADK Runner", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "weather_agent", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: weather_agent", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "sequential_workflow", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: sequential_workflow", - "type": "task" - }, - { - "input": { - "location": "Paris, France" - }, - "metadata": { - "google_adk.tool_call_id": "direct-weather-tool", - "google_adk.tool_name": "get_weather", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "tool: get_weather", - "output": { - "condition": "sunny", - "location": "Paris, France", - "temperature": 72 - }, - "type": "tool" - }, - { - "metadata": { - "google_adk.agent_name": "greeter", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: greeter", - "type": "task" - }, - { - "metadata": { - "google_adk.agent_name": "farewell", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "Agent: farewell", - "type": "task" - } -] diff --git a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.span-events.json b/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.span-events.json deleted file mode 100644 index 656a606a1..000000000 --- a/e2e/scenarios/google-adk-instrumentation/__snapshots__/google-adk-v1000.span-events.json +++ /dev/null @@ -1,171 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-adk-instrumentation" - }, - "metric_keys": [], - "name": "google-adk-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "simple-run" - }, - "metric_keys": [], - "name": "adk-simple-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "sequential-run" - }, - "metric_keys": [], - "name": "adk-sequential-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "metadata": { - "google_adk.session_id": "test-session-1", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Google ADK Runner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "metadata": { - "google_adk.session_id": "test-session-sequential", - "google_adk.user_id": "test-user", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Google ADK Runner", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "weather_agent", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: weather_agent", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "sequential_workflow", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: sequential_workflow", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "google_adk.tool_call_id": "direct-weather-tool", - "google_adk.tool_name": "get_weather", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "tool: get_weather", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "tool" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "greeter", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: greeter", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "metadata": { - "google_adk.agent_name": "farewell", - "model": "gemini-2.5-flash-lite", - "provider": "google-adk" - }, - "metric_keys": [ - "duration" - ], - "name": "Agent: farewell", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } -] diff --git a/e2e/scenarios/google-adk-instrumentation/assertions.ts b/e2e/scenarios/google-adk-instrumentation/assertions.ts index 785900b43..c8b4c22c8 100644 --- a/e2e/scenarios/google-adk-instrumentation/assertions.ts +++ b/e2e/scenarios/google-adk-instrumentation/assertions.ts @@ -1,17 +1,17 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -38,6 +38,16 @@ const VOLATILE_ADK_METRIC_KEYS = new Set([ "prompt_tokens", "tokens", ]); +const SNAPSHOT_ROW_IDENTITY_FIELDS = [ + "org_id", + "project_id", + "experiment_id", + "dataset_id", + "prompt_session_id", + "log_id", + "id", +]; + function isRecord(value: Json | undefined): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -88,20 +98,6 @@ function normalizeADKMetrics(metrics: Json): Json { return normalizeADKVariableTokenCounts(normalized); } -function normalizeADKSummary(summary: Json): Json { - if (!isRecord(summary) || !Array.isArray(summary.metric_keys)) { - return summary; - } - - return { - ...summary, - metric_keys: summary.metric_keys.filter( - (metric): metric is string => - typeof metric === "string" && !VOLATILE_ADK_METRIC_KEYS.has(metric), - ), - } satisfies Json; -} - function normalizeADKOutput(value: Json): Json { const normalized = normalizeADKVariableTokenCounts(value); @@ -121,45 +117,17 @@ function normalizeADKOutput(value: Json): Json { return cloned; } -function dedupeSnapshotItems(items: Json[]): Json[] { - const deduped: Json[] = []; - const seen = new Set(); - - for (const item of items) { - const key = JSON.stringify(item); - if (seen.has(key)) { - continue; - } - seen.add(key); - deduped.push(item); - } +function latestSnapshotEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const eventsByRow = new Map(); - return deduped; -} - -function sortBySpanDepth(events: CapturedLogEvent[]): CapturedLogEvent[] { - const lastById = new Map(); for (const event of events) { - if (event.span.id) { - lastById.set(event.span.id, event); - } - } - - const depthCache = new Map(); - function getDepth(spanId: string | undefined): number { - if (!spanId) return 0; - if (depthCache.has(spanId)) return depthCache.get(spanId)!; - const event = lastById.get(spanId); - if (!event || event.span.parentIds.length === 0) { - depthCache.set(spanId, 0); - return 0; - } - const depth = 1 + getDepth(event.span.parentIds[0]); - depthCache.set(spanId, depth); - return depth; + const key = JSON.stringify( + SNAPSHOT_ROW_IDENTITY_FIELDS.map((field) => event.row[field]), + ); + eventsByRow.set(key, event); } - return [...events].sort((a, b) => getDepth(a.span.id) - getDepth(b.span.id)); + return [...eventsByRow.values()]; } function hasOptionalADKTaskOutput(event: CapturedLogEvent): boolean { @@ -170,26 +138,6 @@ function hasOptionalADKTaskOutput(event: CapturedLogEvent): boolean { ); } -function summarizeADKSpan(event: CapturedLogEvent): Json { - const summary = summarizeWrapperContract(event, [ - "model", - "operation", - "scenario", - "provider", - "google_adk.agent_name", - "google_adk.user_id", - "google_adk.session_id", - "google_adk.tool_name", - "google_adk.tool_call_id", - ]) as Record; - - if (hasOptionalADKTaskOutput(event)) { - delete summary.has_output; - } - - return normalizeADKSummary(summary); -} - function summarizeADKPayload(event: CapturedLogEvent): Json { const metadata = event.row.metadata as Record | undefined; const pickedMetadata: Record = {}; @@ -229,6 +177,31 @@ function summarizeADKPayload(event: CapturedLogEvent): Json { return summary satisfies Json; } +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { + const relevantEvents = latestSnapshotEvents(events).filter( + (event) => + event.span.name !== undefined && + event.span.type !== "llm" && + // Wrapped mode logs an extra start-only tool row. Normalize to the + // terminal tool record so wrapped and auto-hook snapshots stay aligned. + (event.span.type !== "tool" || event.output !== undefined), + ); + + return relevantEvents.map((event) => { + const summary = summarizeADKPayload(event) as Record; + const { name: _name, type: _type, ...fields } = summary; + + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + }, + name: typeof summary.name === "string" ? summary.name : event.span.name, + }; + }); +} + export function defineGoogleADKInstrumentationAssertions(options: { expectLLMSpan: boolean; name: string; @@ -239,11 +212,7 @@ export function defineGoogleADKInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -359,42 +328,8 @@ export function defineGoogleADKInstrumentationAssertions(options: { expect(llmSpan?.metrics).toBeDefined(); }); - test("matches the shared span snapshot", testConfig, async () => { - const relevantEvents = sortBySpanDepth( - events.filter( - (e) => - e.span.name !== undefined && e.span.type !== "llm" && e.span.ended, - ), - ); - const spanSummary = normalizeForSnapshot( - dedupeSnapshotItems( - relevantEvents.map((event) => summarizeADKSpan(event)) as Json[], - ) as Json, - ); - - await matchFileSnapshot( - formatJsonFileSnapshot(spanSummary), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - const relevantEvents = sortBySpanDepth( - events.filter( - (e) => - e.span.name !== undefined && e.span.type !== "llm" && e.span.ended, - ), - ); - const payloadSummary = normalizeForSnapshot( - dedupeSnapshotItems( - relevantEvents.map((event) => summarizeADKPayload(event)) as Json[], - ) as Json, - ); - - await matchFileSnapshot( - formatJsonFileSnapshot(payloadSummary), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/google-adk-instrumentation/scenario.test.ts b/e2e/scenarios/google-adk-instrumentation/scenario.test.ts index 4a1ad2d29..e396bd8f6 100644 --- a/e2e/scenarios/google-adk-instrumentation/scenario.test.ts +++ b/e2e/scenarios/google-adk-instrumentation/scenario.test.ts @@ -51,7 +51,7 @@ describe.concurrent("variants", () => { }); }, expectLLMSpan: false, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-wrapped`, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); @@ -71,7 +71,7 @@ describe.concurrent("variants", () => { }); }, expectLLMSpan: true, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-auto-hook`, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json deleted file mode 100644 index ea0ba486e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json +++ /dev/null @@ -1,454 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-genai-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "generate" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-generate-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "embed" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-embed-operation", - "type": null - }, - { - "input": { - "contents": { - "text": "Paris is the capital of France." - }, - "model": "gemini-embedding-001" - }, - "metadata": { - "model": "gemini-embedding-001" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "embed_content", - "output": { - "embedding_count": 1, - "embedding_length": 3072 - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-attachment-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": "" - }, - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 64, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-return" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-return-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly BONJOUR." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "BONJOUR" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "BONJOUR", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-tool-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json deleted file mode 100644 index 6c19af05e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json +++ /dev/null @@ -1,224 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metric_keys": [], - "name": "google-genai-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "google-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "google-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-embedding-001" - }, - "metric_keys": [ - "duration" - ], - "name": "embed_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "google-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "google-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-return" - }, - "metric_keys": [], - "name": "google-stream-return-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "google-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.json new file mode 100644 index 000000000..0e79e3610 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.json @@ -0,0 +1,755 @@ +{ + "span_tree": [ + { + "name": "google-genai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "google-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "serviceTier": "standard", + "totalTokenCount": 8 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 6, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "google-embed-operation", + "children": [ + { + "name": "embed_content", + "type": "llm", + "children": [], + "input": { + "contents": { + "text": "Paris is the capital of France." + }, + "model": "gemini-embedding-001" + }, + "output": { + "embedding_count": 1, + "embedding_length": 3072 + }, + "metadata": { + "model": "gemini-embedding-001" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "google-attachment-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 48, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 10 + }, + { + "modality": "IMAGE", + "tokenCount": 258 + } + ], + "serviceTier": "standard", + "totalTokenCount": 282 + } + }, + "metadata": { + "maxOutputTokens": 48, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "prompt_tokens": 268, + "tokens": 282 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "google-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 64, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "serviceTier": "standard", + "totalTokenCount": 44 + } + }, + "metadata": { + "maxOutputTokens": 64, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "prompt_tokens": 16, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "google-stream-return-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "serviceTier": "standard", + "totalTokenCount": 9 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 7, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "stream-return", + "testRunId": "" + } + }, + { + "name": "google-grounded-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current estimated population of the city of Paris," + } + ], + "role": "model" + }, + "finishReason": "MAX_TOKENS", + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current population of Paris, France\n Paris population 2024\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2024" + ] + }, + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 36, + "promptTokenCount": 21, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 21 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 305, + "toolUsePromptTokenCount": 110, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 110 + } + ], + "totalTokenCount": 472 + } + }, + "metadata": { + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current population of Paris, France\n Paris population 2024\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2024" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 305, + "completion_tokens": 36, + "duration": 0, + "prompt_tokens": 21, + "tokens": 472 + } + } + ], + "metadata": { + "operation": "grounded-generate", + "testRunId": "" + } + }, + { + "name": "google-grounded-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 142, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + } + } + ], + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 142, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain.", + "usageMetadata": { + "candidatesTokenCount": 63, + "promptTokenCount": 19, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 19 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 146, + "toolUsePromptTokenCount": 74, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 74 + } + ], + "totalTokenCount": 302 + } + }, + "metadata": { + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 142, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 146, + "completion_tokens": 63, + "duration": 0, + "prompt_tokens": 19, + "time_to_first_token": 0, + "tokens": 302 + } + } + ], + "metadata": { + "operation": "grounded-stream", + "testRunId": "" + } + }, + { + "name": "google-tool-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + }, + "metadata": { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.txt b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.txt new file mode 100644 index 000000000..8bc0c0b81 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-tree.txt @@ -0,0 +1,683 @@ +span_tree: +└── google-genai-instrumentation-root [task] + metadata: { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + ├── google-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly PARIS." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "PARIS" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 6, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 6 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 8 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 6, + │ "tokens": 8 + │ } + ├── google-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed_content [llm] + │ input: { + │ "contents": { + │ "text": "Paris is the capital of France." + │ }, + │ "model": "gemini-embedding-001" + │ } + │ output: { + │ "embedding_count": 1, + │ "embedding_length": 3072 + │ } + │ metadata: { + │ "model": "gemini-embedding-001" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── google-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 48, + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "file.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ }, + │ { + │ "text": "Describe the attached image in one short sentence." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 14, + │ "promptTokenCount": 268, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 10 + │ }, + │ { + │ "modality": "IMAGE", + │ "tokenCount": 258 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 282 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 48, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "duration": 0, + │ "prompt_tokens": 268, + │ "tokens": 282 + │ } + ├── google-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 64, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Count from 1 to 3 and include the words one two three." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + │ "usageMetadata": { + │ "candidatesTokenCount": 28, + │ "promptTokenCount": 16, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 16 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 44 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 64, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 28, + │ "duration": 0, + │ "prompt_tokens": 16, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── google-stream-return-operation + │ metadata: { + │ "operation": "stream-return", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly BONJOUR." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "BONJOUR" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "BONJOUR", + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 7, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 7 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 9 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 7, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + ├── google-grounded-generate-operation + │ metadata: { + │ "operation": "grounded-generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current estimated population of the city of Paris," + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "MAX_TOKENS", + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current population of Paris, France\n Paris population 2024\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2024" + │ ] + │ }, + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 36, + │ "promptTokenCount": 21, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 21 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 305, + │ "toolUsePromptTokenCount": 110, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 110 + │ } + │ ], + │ "totalTokenCount": 472 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current population of Paris, France\n Paris population 2024\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2024" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 305, + │ "completion_tokens": 36, + │ "duration": 0, + │ "prompt_tokens": 21, + │ "tokens": 472 + │ } + ├── google-grounded-stream-operation + │ metadata: { + │ "operation": "grounded-stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 142, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ } + │ } + │ ], + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 142, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain.", + │ "usageMetadata": { + │ "candidatesTokenCount": 63, + │ "promptTokenCount": 19, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 19 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 146, + │ "toolUsePromptTokenCount": 74, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 74 + │ } + │ ], + │ "totalTokenCount": 302 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 142, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56 °F (13 °C), feeling like 55 °F (13 °C), and a 42% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 146, + │ "completion_tokens": 63, + │ "duration": 0, + │ "prompt_tokens": 19, + │ "time_to_first_token": 0, + │ "tokens": 302 + │ } + └── google-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── generate_content [llm] + input: { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + } + output: { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + } + metadata: { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + } + metrics: { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json deleted file mode 100644 index ea0ba486e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json +++ /dev/null @@ -1,454 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-genai-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "generate" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-generate-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "embed" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-embed-operation", - "type": null - }, - { - "input": { - "contents": { - "text": "Paris is the capital of France." - }, - "model": "gemini-embedding-001" - }, - "metadata": { - "model": "gemini-embedding-001" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "embed_content", - "output": { - "embedding_count": 1, - "embedding_length": 3072 - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-attachment-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": "" - }, - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 64, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-return" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-return-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly BONJOUR." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "BONJOUR" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "BONJOUR", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-tool-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json deleted file mode 100644 index 6c19af05e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json +++ /dev/null @@ -1,224 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metric_keys": [], - "name": "google-genai-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "google-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "google-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-embedding-001" - }, - "metric_keys": [ - "duration" - ], - "name": "embed_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "google-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "google-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-return" - }, - "metric_keys": [], - "name": "google-stream-return-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "google-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.json new file mode 100644 index 000000000..4b767ac68 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.json @@ -0,0 +1,766 @@ +{ + "span_tree": [ + { + "name": "google-genai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "google-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "serviceTier": "standard", + "totalTokenCount": 8 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 6, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "google-embed-operation", + "children": [ + { + "name": "embed_content", + "type": "llm", + "children": [], + "input": { + "contents": { + "text": "Paris is the capital of France." + }, + "model": "gemini-embedding-001" + }, + "output": { + "embedding_count": 1, + "embedding_length": 3072 + }, + "metadata": { + "model": "gemini-embedding-001" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "google-attachment-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 48, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 10 + }, + { + "modality": "IMAGE", + "tokenCount": 258 + } + ], + "serviceTier": "standard", + "totalTokenCount": 282 + } + }, + "metadata": { + "maxOutputTokens": 48, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "prompt_tokens": 268, + "tokens": 282 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "google-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 64, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "serviceTier": "standard", + "totalTokenCount": 44 + } + }, + "metadata": { + "maxOutputTokens": 64, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "prompt_tokens": 16, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "google-stream-return-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "serviceTier": "standard", + "totalTokenCount": 9 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 7, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "stream-return", + "testRunId": "" + } + }, + { + "name": "google-grounded-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The estimated population of the city of Paris is 2,047,602 as of January 2026. [cite: 8" + }, + { + "text": "The estimated population of the city of Paris is 2,047,602 as of January 2026. [cite: 8" + } + ], + "role": "model" + }, + "finishReason": "MAX_TOKENS", + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2024\n current population of Paris, France\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2024" + ] + }, + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "cacheTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 2 + } + ], + "cachedContentTokenCount": 2, + "candidatesTokenCount": 92, + "promptTokenCount": 21, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 21 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 282, + "toolUsePromptTokenCount": 110, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 110 + } + ], + "totalTokenCount": 505 + } + }, + "metadata": { + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2024\n current population of Paris, France\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2024" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 282, + "completion_tokens": 92, + "duration": 0, + "prompt_cached_tokens": 2, + "prompt_tokens": 21, + "tokens": 505 + } + } + ], + "metadata": { + "operation": "grounded-generate", + "testRunId": "" + } + }, + { + "name": "google-grounded-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 138, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + } + } + ], + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 138, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain.", + "usageMetadata": { + "candidatesTokenCount": 63, + "promptTokenCount": 19, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 19 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 213, + "toolUsePromptTokenCount": 74, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 74 + } + ], + "totalTokenCount": 369 + } + }, + "metadata": { + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 138, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 213, + "completion_tokens": 63, + "duration": 0, + "prompt_tokens": 19, + "time_to_first_token": 0, + "tokens": 369 + } + } + ], + "metadata": { + "operation": "grounded-stream", + "testRunId": "" + } + }, + { + "name": "google-tool-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + }, + "metadata": { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.txt b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.txt new file mode 100644 index 000000000..8c5aef938 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-tree.txt @@ -0,0 +1,694 @@ +span_tree: +└── google-genai-instrumentation-root [task] + metadata: { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + ├── google-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly PARIS." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "PARIS" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 6, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 6 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 8 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 6, + │ "tokens": 8 + │ } + ├── google-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed_content [llm] + │ input: { + │ "contents": { + │ "text": "Paris is the capital of France." + │ }, + │ "model": "gemini-embedding-001" + │ } + │ output: { + │ "embedding_count": 1, + │ "embedding_length": 3072 + │ } + │ metadata: { + │ "model": "gemini-embedding-001" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── google-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 48, + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "file.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ }, + │ { + │ "text": "Describe the attached image in one short sentence." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 14, + │ "promptTokenCount": 268, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 10 + │ }, + │ { + │ "modality": "IMAGE", + │ "tokenCount": 258 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 282 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 48, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "duration": 0, + │ "prompt_tokens": 268, + │ "tokens": 282 + │ } + ├── google-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 64, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Count from 1 to 3 and include the words one two three." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + │ "usageMetadata": { + │ "candidatesTokenCount": 28, + │ "promptTokenCount": 16, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 16 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 44 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 64, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 28, + │ "duration": 0, + │ "prompt_tokens": 16, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── google-stream-return-operation + │ metadata: { + │ "operation": "stream-return", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly BONJOUR." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "BONJOUR" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "BONJOUR", + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 7, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 7 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 9 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 7, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + ├── google-grounded-generate-operation + │ metadata: { + │ "operation": "grounded-generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The estimated population of the city of Paris is 2,047,602 as of January 2026. [cite: 8" + │ }, + │ { + │ "text": "The estimated population of the city of Paris is 2,047,602 as of January 2026. [cite: 8" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "MAX_TOKENS", + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2024\n current population of Paris, France\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2024" + │ ] + │ }, + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "cacheTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 2 + │ } + │ ], + │ "cachedContentTokenCount": 2, + │ "candidatesTokenCount": 92, + │ "promptTokenCount": 21, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 21 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 282, + │ "toolUsePromptTokenCount": 110, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 110 + │ } + │ ], + │ "totalTokenCount": 505 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2024\n current population of Paris, France\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2024" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 282, + │ "completion_tokens": 92, + │ "duration": 0, + │ "prompt_cached_tokens": 2, + │ "prompt_tokens": 21, + │ "tokens": 505 + │ } + ├── google-grounded-stream-operation + │ metadata: { + │ "operation": "grounded-stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 138, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ } + │ } + │ ], + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 138, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain.", + │ "usageMetadata": { + │ "candidatesTokenCount": 63, + │ "promptTokenCount": 19, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 19 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 213, + │ "toolUsePromptTokenCount": 74, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 74 + │ } + │ ], + │ "totalTokenCount": 369 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 138, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C), feeling like 55°F (13°C), and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 213, + │ "completion_tokens": 63, + │ "duration": 0, + │ "prompt_tokens": 19, + │ "time_to_first_token": 0, + │ "tokens": 369 + │ } + └── google-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── generate_content [llm] + input: { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + } + output: { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + } + metadata: { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + } + metrics: { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json deleted file mode 100644 index ea0ba486e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json +++ /dev/null @@ -1,454 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-genai-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "generate" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-generate-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "embed" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-embed-operation", - "type": null - }, - { - "input": { - "contents": { - "text": "Paris is the capital of France." - }, - "model": "gemini-embedding-001" - }, - "metadata": { - "model": "gemini-embedding-001" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "embed_content", - "output": { - "embedding_count": 1, - "embedding_length": 3072 - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-attachment-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": "" - }, - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 64, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-return" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-return-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly BONJOUR." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "BONJOUR" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "BONJOUR", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-tool-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json deleted file mode 100644 index 6c19af05e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json +++ /dev/null @@ -1,224 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metric_keys": [], - "name": "google-genai-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "google-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "google-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-embedding-001" - }, - "metric_keys": [ - "duration" - ], - "name": "embed_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "google-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "google-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-return" - }, - "metric_keys": [], - "name": "google-stream-return-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "google-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.json new file mode 100644 index 000000000..676ccdf9f --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.json @@ -0,0 +1,765 @@ +{ + "span_tree": [ + { + "name": "google-genai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "google-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "serviceTier": "standard", + "totalTokenCount": 8 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 6, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "google-embed-operation", + "children": [ + { + "name": "embed_content", + "type": "llm", + "children": [], + "input": { + "contents": { + "text": "Paris is the capital of France." + }, + "model": "gemini-embedding-001" + }, + "output": { + "embedding_count": 1, + "embedding_length": 3072 + }, + "metadata": { + "model": "gemini-embedding-001" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "google-attachment-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 48, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 10 + }, + { + "modality": "IMAGE", + "tokenCount": 258 + } + ], + "serviceTier": "standard", + "totalTokenCount": 282 + } + }, + "metadata": { + "maxOutputTokens": 48, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "prompt_tokens": 268, + "tokens": 282 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "google-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 64, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "serviceTier": "standard", + "totalTokenCount": 44 + } + }, + "metadata": { + "maxOutputTokens": 64, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "prompt_tokens": 16, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "google-stream-return-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "serviceTier": "standard", + "totalTokenCount": 9 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 7, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "stream-return", + "testRunId": "" + } + }, + { + "name": "google-grounded-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current estimated population of the city of" + } + ], + "role": "model" + }, + "finishReason": "MAX_TOKENS", + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2026", + "Paris population 2025" + ] + }, + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "cacheTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 2 + } + ], + "cachedContentTokenCount": 2, + "candidatesTokenCount": 43, + "promptTokenCount": 21, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 21 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 307, + "toolUsePromptTokenCount": 119, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 119 + } + ], + "totalTokenCount": 490 + } + }, + "metadata": { + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2026", + "Paris population 2025" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 307, + "completion_tokens": 43, + "duration": 0, + "prompt_cached_tokens": 2, + "prompt_tokens": 21, + "tokens": 490 + } + } + ], + "metadata": { + "operation": "grounded-generate", + "testRunId": "" + } + }, + { + "name": "google-grounded-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 108, + "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + } + } + ], + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 108, + "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain.", + "usageMetadata": { + "candidatesTokenCount": 49, + "promptTokenCount": 19, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 19 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 65, + "toolUsePromptTokenCount": 73, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 73 + } + ], + "totalTokenCount": 206 + } + }, + "metadata": { + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 108, + "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 65, + "completion_tokens": 49, + "duration": 0, + "prompt_tokens": 19, + "time_to_first_token": 0, + "tokens": 206 + } + } + ], + "metadata": { + "operation": "grounded-stream", + "testRunId": "" + } + }, + { + "name": "google-tool-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + }, + "metadata": { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.txt b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.txt new file mode 100644 index 000000000..f0eedf100 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-tree.txt @@ -0,0 +1,693 @@ +span_tree: +└── google-genai-instrumentation-root [task] + metadata: { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + ├── google-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly PARIS." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "PARIS" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 6, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 6 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 8 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 6, + │ "tokens": 8 + │ } + ├── google-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed_content [llm] + │ input: { + │ "contents": { + │ "text": "Paris is the capital of France." + │ }, + │ "model": "gemini-embedding-001" + │ } + │ output: { + │ "embedding_count": 1, + │ "embedding_length": 3072 + │ } + │ metadata: { + │ "model": "gemini-embedding-001" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── google-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 48, + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "file.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ }, + │ { + │ "text": "Describe the attached image in one short sentence." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 14, + │ "promptTokenCount": 268, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 10 + │ }, + │ { + │ "modality": "IMAGE", + │ "tokenCount": 258 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 282 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 48, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "duration": 0, + │ "prompt_tokens": 268, + │ "tokens": 282 + │ } + ├── google-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 64, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Count from 1 to 3 and include the words one two three." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + │ "usageMetadata": { + │ "candidatesTokenCount": 28, + │ "promptTokenCount": 16, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 16 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 44 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 64, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 28, + │ "duration": 0, + │ "prompt_tokens": 16, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── google-stream-return-operation + │ metadata: { + │ "operation": "stream-return", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly BONJOUR." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "BONJOUR" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "BONJOUR", + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 7, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 7 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 9 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 7, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + ├── google-grounded-generate-operation + │ metadata: { + │ "operation": "grounded-generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current estimated population of the city of" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "MAX_TOKENS", + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2026", + │ "Paris population 2025" + │ ] + │ }, + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "cacheTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 2 + │ } + │ ], + │ "cachedContentTokenCount": 2, + │ "candidatesTokenCount": 43, + │ "promptTokenCount": 21, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 21 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 307, + │ "toolUsePromptTokenCount": 119, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 119 + │ } + │ ], + │ "totalTokenCount": 490 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2026", + │ "Paris population 2025" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 307, + │ "completion_tokens": 43, + │ "duration": 0, + │ "prompt_cached_tokens": 2, + │ "prompt_tokens": 21, + │ "tokens": 490 + │ } + ├── google-grounded-stream-operation + │ metadata: { + │ "operation": "grounded-stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 108, + │ "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ } + │ } + │ ], + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 108, + │ "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain.", + │ "usageMetadata": { + │ "candidatesTokenCount": 49, + │ "promptTokenCount": 19, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 19 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 65, + │ "toolUsePromptTokenCount": 73, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 73 + │ } + │ ], + │ "totalTokenCount": 206 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 108, + │ "text": "The current weather in Paris, France is cloudy with a temperature of 56°F (13°C) and a 43% chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 65, + │ "completion_tokens": 49, + │ "duration": 0, + │ "prompt_tokens": 19, + │ "time_to_first_token": 0, + │ "tokens": 206 + │ } + └── google-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── generate_content [llm] + input: { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + } + output: { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + } + metadata: { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + } + metrics: { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json deleted file mode 100644 index ea0ba486e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json +++ /dev/null @@ -1,454 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-genai-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "generate" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-generate-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "embed" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-embed-operation", - "type": null - }, - { - "input": { - "contents": { - "text": "Paris is the capital of France." - }, - "model": "gemini-embedding-001" - }, - "metadata": { - "model": "gemini-embedding-001" - }, - "metrics": { - "duration": 0, - "end": 0, - "start": 0 - }, - "name": "embed_content", - "output": { - "embedding_count": 1, - "embedding_length": 3072 - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-attachment-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": "" - }, - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 64, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-return" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-return-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly BONJOUR." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "tokens": "" - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "BONJOUR" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "BONJOUR", - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-tool-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": "", - "duration": 0, - "end": 0, - "prompt_tokens": "", - "start": 0, - "tokens": "" - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "connection": "keep-alive", - "content-type": "application/json; charset=UTF-8", - "date": "", - "keep-alive": "timeout=5", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": "", - "promptTokenCount": "", - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": "" - } - ], - "totalTokenCount": "" - } - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json deleted file mode 100644 index 6c19af05e..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json +++ /dev/null @@ -1,224 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metric_keys": [], - "name": "google-genai-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "google-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embed" - }, - "metric_keys": [], - "name": "google-embed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-embedding-001" - }, - "metric_keys": [ - "duration" - ], - "name": "embed_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "google-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "google-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-return" - }, - "metric_keys": [], - "name": "google-stream-return-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "google-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.json new file mode 100644 index 000000000..df5b18f95 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.json @@ -0,0 +1,760 @@ +{ + "span_tree": [ + { + "name": "google-genai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "google-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "serviceTier": "standard", + "totalTokenCount": 8 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 6, + "tokens": 8 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "google-embed-operation", + "children": [ + { + "name": "embed_content", + "type": "llm", + "children": [], + "input": { + "contents": { + "text": "Paris is the capital of France." + }, + "model": "gemini-embedding-001" + }, + "output": { + "embedding_count": 1, + "embedding_length": 3072 + }, + "metadata": { + "model": "gemini-embedding-001" + }, + "metrics": { + "duration": 0 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "google-attachment-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 48, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 10 + }, + { + "modality": "IMAGE", + "tokenCount": 258 + } + ], + "serviceTier": "standard", + "totalTokenCount": 282 + } + }, + "metadata": { + "maxOutputTokens": 48, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "prompt_tokens": 268, + "tokens": 282 + } + } + ], + "metadata": { + "operation": "attachment", + "testRunId": "" + } + }, + { + "name": "google-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 64, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "serviceTier": "standard", + "totalTokenCount": 44 + } + }, + "metadata": { + "maxOutputTokens": 64, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "prompt_tokens": 16, + "time_to_first_token": 0, + "tokens": 44 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "google-stream-return-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "serviceTier": "standard", + "totalTokenCount": 9 + } + }, + "metadata": { + "maxOutputTokens": 24, + "model": "gemini-2.5-flash-lite", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "prompt_tokens": 7, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "stream-return", + "testRunId": "" + } + }, + { + "name": "google-grounded-generate-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current estimated population of the city of Paris, France, is 2,047,602 as of January 2026. [" + }, + { + "text": "The current estimated population of the city of Paris, France, is 2,047,602 as of January 2026. [" + } + ], + "role": "model" + }, + "finishReason": "MAX_TOKENS", + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2026", + "Paris population 2025" + ] + }, + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 101, + "promptTokenCount": 21, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 21 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 282, + "toolUsePromptTokenCount": 119, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 119 + } + ], + "totalTokenCount": 523 + } + }, + "metadata": { + "groundingMetadata": { + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + }, + "webSearchQueries": [ + "current population of Paris, France", + "Paris population 2026", + "Paris population 2025" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 282, + "completion_tokens": 101, + "duration": 0, + "prompt_tokens": 21, + "tokens": 523 + } + } + ], + "metadata": { + "operation": "grounded-generate", + "testRunId": "" + } + }, + { + "name": "google-grounded-stream-operation", + "children": [ + { + "name": "generate_content_stream", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 256, + "temperature": 0 + }, + "contents": { + "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + }, + "model": "gemini-2.5-flash" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 105, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + } + } + ], + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 105, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain.", + "usageMetadata": { + "candidatesTokenCount": 46, + "promptTokenCount": 19, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 19 + } + ], + "serviceTier": "standard", + "thoughtsTokenCount": 95, + "toolUsePromptTokenCount": 73, + "toolUsePromptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 73 + } + ], + "totalTokenCount": 233 + } + }, + "metadata": { + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "title": "Weather information for Paris, FR", + "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + } + } + ], + "groundingSupports": [ + { + "groundingChunkIndices": [ + 0 + ], + "segment": { + "endIndex": 105, + "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + } + } + ], + "searchEntryPoint": { + "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + }, + "webSearchQueries": [ + "current weather in Paris" + ] + }, + "maxOutputTokens": 256, + "model": "gemini-2.5-flash", + "temperature": 0, + "tools": [ + { + "googleSearch": {} + } + ] + }, + "metrics": { + "completion_reasoning_tokens": 95, + "completion_tokens": 46, + "duration": 0, + "prompt_tokens": 19, + "time_to_first_token": 0, + "tokens": 233 + } + } + ], + "metadata": { + "operation": "grounded-stream", + "testRunId": "" + } + }, + { + "name": "google-tool-operation", + "children": [ + { + "name": "generate_content", + "type": "llm", + "children": [], + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + }, + "metadata": { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.txt b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.txt new file mode 100644 index 000000000..cd2726eca --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-tree.txt @@ -0,0 +1,688 @@ +span_tree: +└── google-genai-instrumentation-root [task] + metadata: { + "scenario": "google-genai-instrumentation", + "testRunId": "" + } + ├── google-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly PARIS." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "PARIS" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 6, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 6 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 8 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 6, + │ "tokens": 8 + │ } + ├── google-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── embed_content [llm] + │ input: { + │ "contents": { + │ "text": "Paris is the capital of France." + │ }, + │ "model": "gemini-embedding-001" + │ } + │ output: { + │ "embedding_count": 1, + │ "embedding_length": 3072 + │ } + │ metadata: { + │ "model": "gemini-embedding-001" + │ } + │ metrics: { + │ "duration": 0 + │ } + ├── google-attachment-operation + │ metadata: { + │ "operation": "attachment", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 48, + │ "temperature": 0 + │ }, + │ "contents": [ + │ { + │ "parts": [ + │ { + │ "image_url": { + │ "url": { + │ "content_type": "image/png", + │ "filename": "file.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ }, + │ { + │ "text": "Describe the attached image in one short sentence." + │ } + │ ], + │ "role": "user" + │ } + │ ], + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash-lite", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 14, + │ "promptTokenCount": 268, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 10 + │ }, + │ { + │ "modality": "IMAGE", + │ "tokenCount": 258 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 282 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 48, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "duration": 0, + │ "prompt_tokens": 268, + │ "tokens": 282 + │ } + ├── google-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 64, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Count from 1 to 3 and include the words one two three." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + │ "usageMetadata": { + │ "candidatesTokenCount": 28, + │ "promptTokenCount": 16, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 16 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 44 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 64, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 28, + │ "duration": 0, + │ "prompt_tokens": 16, + │ "time_to_first_token": 0, + │ "tokens": 44 + │ } + ├── google-stream-return-operation + │ metadata: { + │ "operation": "stream-return", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 24, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Reply with exactly BONJOUR." + │ }, + │ "model": "gemini-2.5-flash-lite" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "BONJOUR" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP" + │ } + │ ], + │ "text": "BONJOUR", + │ "usageMetadata": { + │ "candidatesTokenCount": 2, + │ "promptTokenCount": 7, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 7 + │ } + │ ], + │ "serviceTier": "standard", + │ "totalTokenCount": 9 + │ } + │ } + │ metadata: { + │ "maxOutputTokens": 24, + │ "model": "gemini-2.5-flash-lite", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "duration": 0, + │ "prompt_tokens": 7, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + ├── google-grounded-generate-operation + │ metadata: { + │ "operation": "grounded-generate", + │ "testRunId": "" + │ } + │ └── generate_content [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current population of Paris, France?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current estimated population of the city of Paris, France, is 2,047,602 as of January 2026. [" + │ }, + │ { + │ "text": "The current estimated population of the city of Paris, France, is 2,047,602 as of January 2026. [" + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "MAX_TOKENS", + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2026", + │ "Paris population 2025" + │ ] + │ }, + │ "index": 0 + │ } + │ ], + │ "modelVersion": "gemini-2.5-flash", + │ "responseId": "", + │ "sdkHttpResponse": { + │ "headers": { + │ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + │ "connection": "keep-alive", + │ "content-type": "application/json; charset=UTF-8", + │ "date": "", + │ "keep-alive": "timeout=5", + │ "server": "scaffolding on HTTPServer2", + │ "server-timing": "", + │ "transfer-encoding": "chunked", + │ "vary": "Origin, X-Origin, Referer", + │ "x-content-type-options": "nosniff", + │ "x-frame-options": "SAMEORIGIN", + │ "x-gemini-service-tier": "", + │ "x-xss-protection": "0" + │ } + │ }, + │ "usageMetadata": { + │ "candidatesTokenCount": 101, + │ "promptTokenCount": 21, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 21 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 282, + │ "toolUsePromptTokenCount": 119, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 119 + │ } + │ ], + │ "totalTokenCount": 523 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n Paris population 2026\n current population of Paris, France\n Paris population 2025\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current population of Paris, France", + │ "Paris population 2026", + │ "Paris population 2025" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 282, + │ "completion_tokens": 101, + │ "duration": 0, + │ "prompt_tokens": 21, + │ "tokens": 523 + │ } + ├── google-grounded-stream-operation + │ metadata: { + │ "operation": "grounded-stream", + │ "testRunId": "" + │ } + │ └── generate_content_stream [llm] + │ input: { + │ "config": { + │ "maxOutputTokens": 256, + │ "temperature": 0 + │ }, + │ "contents": { + │ "text": "Use Google Search grounding and answer in one sentence: What is the current weather in Paris?" + │ }, + │ "model": "gemini-2.5-flash" + │ } + │ output: { + │ "candidates": [ + │ { + │ "content": { + │ "parts": [ + │ { + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + │ } + │ ], + │ "role": "model" + │ }, + │ "finishReason": "STOP", + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 105, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ } + │ } + │ ], + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 105, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain.", + │ "usageMetadata": { + │ "candidatesTokenCount": 46, + │ "promptTokenCount": 19, + │ "promptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 19 + │ } + │ ], + │ "serviceTier": "standard", + │ "thoughtsTokenCount": 95, + │ "toolUsePromptTokenCount": 73, + │ "toolUsePromptTokensDetails": [ + │ { + │ "modality": "TEXT", + │ "tokenCount": 73 + │ } + │ ], + │ "totalTokenCount": 233 + │ } + │ } + │ metadata: { + │ "groundingMetadata": { + │ "groundingChunks": [ + │ { + │ "web": { + │ "title": "Weather information for Paris, FR", + │ "uri": "https://www.google.com/search?q=weather+in+Paris,+FR" + │ } + │ } + │ ], + │ "groundingSupports": [ + │ { + │ "groundingChunkIndices": [ + │ 0 + │ ], + │ "segment": { + │ "endIndex": 105, + │ "text": "The current weather in Paris, France, is cloudy with a temperature of 56°F (13°C) and a chance of rain." + │ } + │ } + │ ], + │ "searchEntryPoint": { + │ "renderedContent": "\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
\n
\n current weather in Paris\n
\n
\n" + │ }, + │ "webSearchQueries": [ + │ "current weather in Paris" + │ ] + │ }, + │ "maxOutputTokens": 256, + │ "model": "gemini-2.5-flash", + │ "temperature": 0, + │ "tools": [ + │ { + │ "googleSearch": {} + │ } + │ ] + │ } + │ metrics: { + │ "completion_reasoning_tokens": 95, + │ "completion_tokens": 46, + │ "duration": 0, + │ "prompt_tokens": 19, + │ "time_to_first_token": 0, + │ "tokens": 233 + │ } + └── google-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── generate_content [llm] + input: { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + } + output: { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "connection": "keep-alive", + "content-type": "application/json; charset=UTF-8", + "date": "", + "keep-alive": "timeout=5", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "serviceTier": "standard", + "totalTokenCount": 82 + } + } + metadata: { + "maxOutputTokens": 128, + "model": "gemini-2.5-flash-lite", + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parametersJsonSchema": { + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + } + ] + } + ] + } + metrics: { + "completion_tokens": 17, + "duration": 0, + "prompt_tokens": 65, + "tokens": 82 + } diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json deleted file mode 100644 index 38006e222..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json +++ /dev/null @@ -1,418 +0,0 @@ -[ - { - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-genai-instrumentation-root", - "type": "task" - }, - { - "metadata": { - "operation": "generate" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-generate-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 16, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": 2, - "duration": 0, - "end": 0, - "prompt_tokens": 6, - "start": 0, - "tokens": 8 - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 2, - "promptTokenCount": 6, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 6 - } - ], - "totalTokenCount": 8 - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "attachment" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-attachment-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": 14, - "duration": 0, - "end": 0, - "prompt_tokens": 268, - "start": 0, - "tokens": 282 - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 14, - "promptTokenCount": 268, - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": 258 - }, - { - "modality": "TEXT", - "tokenCount": 10 - } - ], - "totalTokenCount": 282 - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 32, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": 28, - "duration": 0, - "end": 0, - "prompt_tokens": 16, - "start": 0, - "time_to_first_token": 0, - "tokens": 44 - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": 28, - "promptTokenCount": 16, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 16 - } - ], - "totalTokenCount": 44 - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "stream-return" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-stream-return-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 16, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly BONJOUR." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": 2, - "duration": 0, - "end": 0, - "prompt_tokens": 7, - "start": 0, - "time_to_first_token": 0, - "tokens": 9 - }, - "name": "generate_content_stream", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "BONJOUR" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "BONJOUR", - "usageMetadata": { - "candidatesTokenCount": 2, - "promptTokenCount": 7, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 7 - } - ], - "totalTokenCount": 9 - } - }, - "type": "llm" - }, - { - "metadata": { - "operation": "tool" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "name": "google-tool-operation", - "type": null - }, - { - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metrics": { - "completion_tokens": 17, - "duration": 0, - "end": 0, - "prompt_tokens": 65, - "start": 0, - "tokens": 82 - }, - "name": "generate_content", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 17, - "promptTokenCount": 65, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 65 - } - ], - "totalTokenCount": 82 - } - }, - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json deleted file mode 100644 index dacc75405..000000000 --- a/e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json +++ /dev/null @@ -1,192 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "google-genai-instrumentation" - }, - "metric_keys": [], - "name": "google-genai-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "generate" - }, - "metric_keys": [], - "name": "google-generate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "attachment" - }, - "metric_keys": [], - "name": "google-attachment-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "google-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-return" - }, - "metric_keys": [], - "name": "google-stream-return-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "generate_content_stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "google-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite" - }, - "metric_keys": [ - "completion_tokens", - "duration", - "prompt_tokens", - "tokens" - ], - "name": "generate_content", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/google-genai-instrumentation/assertions.ts b/e2e/scenarios/google-genai-instrumentation/assertions.ts index 979943c43..42db133f7 100644 --- a/e2e/scenarios/google-genai-instrumentation/assertions.ts +++ b/e2e/scenarios/google-genai-instrumentation/assertions.ts @@ -1,22 +1,22 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { effectiveScenarioTimeoutMs, withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findChildSpans, findLatestChildSpan, findLatestSpan, } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { GOOGLE_EMBEDDING_MODEL, @@ -256,19 +256,6 @@ function normalizeGoogleOutput(event: CapturedLogEvent): Json { ); } -function normalizeGoogleSummary(summary: Json): Json { - if (!isRecord(summary) || !Array.isArray(summary.metric_keys)) { - return summary; - } - - return { - ...summary, - metric_keys: summary.metric_keys.filter( - (metric): metric is string => metric !== "prompt_cached_tokens", - ), - } satisfies Json; -} - function summarizeGooglePayload(event: CapturedLogEvent): Json { return { input: event.input as Json, @@ -332,22 +319,20 @@ function buildRelevantEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { ].map((event) => event!); } -function buildSpanSummary(events: CapturedLogEvent[]): Json { - return normalizeForSnapshot( - buildRelevantEvents(events).map((event) => - normalizeGoogleSummary( - summarizeWrapperContract(event, ["model", "operation", "scenario"]), - ), - ) as Json, - ); -} +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { + return buildRelevantEvents(events).map((event) => { + const summary = summarizeGooglePayload(event) as Record; + const { name: _name, type: _type, ...fields } = summary; -function buildPayloadSummary(events: CapturedLogEvent[]): Json { - return normalizeForSnapshot( - buildRelevantEvents(events).map((event) => - summarizeGooglePayload(event), - ) as Json, - ); + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + }, + name: typeof summary.name === "string" ? summary.name : event.span.name, + }; + }); } export function defineGoogleGenAIInstrumentationAssertions(options: { @@ -359,11 +344,7 @@ export function defineGoogleGenAIInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); const testConfig = { @@ -639,18 +620,8 @@ export function defineGoogleGenAIInstrumentationAssertions(options: { ).toBe(true); }); - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events)), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildPayloadSummary(events)), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-events.json b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-events.json deleted file mode 100644 index f0d1c924a..000000000 --- a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-events.json +++ /dev/null @@ -1,181 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "groq-instrumentation" - }, - "metric_keys": [], - "name": "groq-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "groq-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "groq-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "reasoning-stream" - }, - "metric_keys": [], - "name": "groq-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "qwen/qwen3-32b", - "provider": "groq", - "reasoning_format": "parsed", - "temperature": 0.6 - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "groq-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.json b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.json new file mode 100644 index 000000000..1a03c7001 --- /dev/null +++ b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.json @@ -0,0 +1,241 @@ +{ + "span_tree": [ + { + "name": "groq-instrumentation-root", + "type": "task", + "children": [ + { + "name": "groq-chat-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "OK", + "role": "assistant" + } + } + ], + "metadata": { + "max_completion_tokens": 12, + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0 + }, + "metrics": { + "completion_time": 0.00745245, + "completion_tokens": 2, + "prompt_time": 0.001977594, + "prompt_tokens": 40, + "queue_time": 0.095205932, + "time_to_first_token": 0, + "tokens": 42, + "total_time": 0.009430044 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "groq-stream-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_time": 0.006624354, + "completion_tokens": 2, + "prompt_time": 0.017750957, + "prompt_tokens": 40, + "queue_time": 0.183890233, + "time_to_first_token": 0, + "tokens": 42, + "total_time": 0.024375311 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "groq-reasoning-stream-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Solve this step by step: Elena has 3 boxes with 4 marbles each, gives away 5 marbles, then doubles what remains. Reply with just the final number.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "14", + "reasoning": "Okay, let's see. Elena has 3 boxes, each with 4 marbles. So first, I need to figure out how many marbles she has in total. If each box has 4 marbles and there are 3 boxes, then I multiply 3 by 4. 3 times 4 is 12. So she starts with 12 marbles.\n\nThen she gives away 5 marbles. So I need to subtract 5 from 12. Let me do that: 12 minus 5 equals 7. So after giving away 5 marbles, she has 7 left.\n\nNext, she doubles what remains. Doubling means multiplying by 2. So 7 times 2 is 14. Let me check that again. 7 marbles doubled would be 14. \n\nWait, let me go through each step once more to make sure I didn't make a mistake. Starting with 3 boxes times 4 marbles each: 3*4=12. Then subtracting 5: 12-5=7. Then doubling that: 7*2=14. Yeah, that seems right. I don't think I messed up any numbers here. So the final number should be 14.\n", + "role": "assistant" + } + } + ], + "metadata": { + "max_completion_tokens": 512, + "model": "qwen/qwen3-32b", + "provider": "groq", + "reasoning_format": "parsed", + "stream": true, + "temperature": 0.6 + }, + "metrics": { + "completion_reasoning_tokens": 268, + "completion_time": 0.454665574, + "completion_tokens": 275, + "prompt_time": 0.004045298, + "prompt_tokens": 46, + "queue_time": 0.046866887, + "time_to_first_token": 0, + "tokens": 321, + "total_time": 0.458710872 + } + } + ], + "metadata": { + "operation": "reasoning-stream", + "testRunId": "" + } + }, + { + "name": "groq-tool-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Check the weather in Vienna and use the weather tool.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "City name.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_time": 0.036719131, + "completion_tokens": 11, + "prompt_time": 0.09047757, + "prompt_tokens": 246, + "queue_time": 0.327464415, + "time_to_first_token": 0, + "tokens": 257, + "total_time": 0.127196701 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "groq-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.txt b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.txt new file mode 100644 index 000000000..032f0e8c6 --- /dev/null +++ b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-auto.span-tree.txt @@ -0,0 +1,201 @@ +span_tree: +└── groq-instrumentation-root [task] + metadata: { + "scenario": "groq-instrumentation", + "testRunId": "" + } + ├── groq-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "OK", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_completion_tokens": 12, + │ "model": "llama-3.3-70b-versatile", + │ "provider": "groq", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_time": 0.00745245, + │ "completion_tokens": 2, + │ "prompt_time": 0.001977594, + │ "prompt_tokens": 40, + │ "queue_time": 0.095205932, + │ "time_to_first_token": 0, + │ "tokens": 42, + │ "total_time": 0.009430044 + │ } + ├── groq-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "llama-3.3-70b-versatile", + │ "provider": "groq", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_time": 0.006624354, + │ "completion_tokens": 2, + │ "prompt_time": 0.017750957, + │ "prompt_tokens": 40, + │ "queue_time": 0.183890233, + │ "time_to_first_token": 0, + │ "tokens": 42, + │ "total_time": 0.024375311 + │ } + ├── groq-reasoning-stream-operation + │ metadata: { + │ "operation": "reasoning-stream", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Solve this step by step: Elena has 3 boxes with 4 marbles each, gives away 5 marbles, then doubles what remains. Reply with just the final number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "14", + │ "reasoning": "Okay, let's see. Elena has 3 boxes, each with 4 marbles. So first, I need to figure out how many marbles she has in total. If each box has 4 marbles and there are 3 boxes, then I multiply 3 by 4. 3 times 4 is 12. So she starts with 12 marbles.\n\nThen she gives away 5 marbles. So I need to subtract 5 from 12. Let me do that: 12 minus 5 equals 7. So after giving away 5 marbles, she has 7 left.\n\nNext, she doubles what remains. Doubling means multiplying by 2. So 7 times 2 is 14. Let me check that again. 7 marbles doubled would be 14. \n\nWait, let me go through each step once more to make sure I didn't make a mistake. Starting with 3 boxes times 4 marbles each: 3*4=12. Then subtracting 5: 12-5=7. Then doubling that: 7*2=14. Yeah, that seems right. I don't think I messed up any numbers here. So the final number should be 14.\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_completion_tokens": 512, + │ "model": "qwen/qwen3-32b", + │ "provider": "groq", + │ "reasoning_format": "parsed", + │ "stream": true, + │ "temperature": 0.6 + │ } + │ metrics: { + │ "completion_reasoning_tokens": 268, + │ "completion_time": 0.454665574, + │ "completion_tokens": 275, + │ "prompt_time": 0.004045298, + │ "prompt_tokens": 46, + │ "queue_time": 0.046866887, + │ "time_to_first_token": 0, + │ "tokens": 321, + │ "total_time": 0.458710872 + │ } + └── groq-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── groq.chat.completions.create [llm] + input: [ + { + "content": "Check the weather in Vienna and use the weather tool.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ] + metadata: { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "City name.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + } + metrics: { + "completion_time": 0.036719131, + "completion_tokens": 11, + "prompt_time": 0.09047757, + "prompt_tokens": 246, + "queue_time": 0.327464415, + "time_to_first_token": 0, + "tokens": 257, + "total_time": 0.127196701 + } diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-events.json b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-events.json deleted file mode 100644 index f0d1c924a..000000000 --- a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-events.json +++ /dev/null @@ -1,181 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "groq-instrumentation" - }, - "metric_keys": [], - "name": "groq-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "groq-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "groq-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "reasoning-stream" - }, - "metric_keys": [], - "name": "groq-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "qwen/qwen3-32b", - "provider": "groq", - "reasoning_format": "parsed", - "temperature": 0.6 - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "groq-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "llama-3.3-70b-versatile", - "provider": "groq", - "temperature": 0 - }, - "metric_keys": [ - "completion_time", - "completion_tokens", - "prompt_time", - "prompt_tokens", - "queue_time", - "time_to_first_token", - "tokens", - "total_time" - ], - "name": "groq.chat.completions.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.json b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.json new file mode 100644 index 000000000..963acc6ef --- /dev/null +++ b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.json @@ -0,0 +1,241 @@ +{ + "span_tree": [ + { + "name": "groq-instrumentation-root", + "type": "task", + "children": [ + { + "name": "groq-chat-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "OK", + "role": "assistant" + } + } + ], + "metadata": { + "max_completion_tokens": 12, + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0 + }, + "metrics": { + "completion_time": 0.006650543, + "completion_tokens": 2, + "prompt_time": 0.002703216, + "prompt_tokens": 40, + "queue_time": 0.109831551, + "time_to_first_token": 0, + "tokens": 42, + "total_time": 0.009353759 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "groq-stream-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_time": 0.007875603, + "completion_tokens": 2, + "prompt_time": 0.001919959, + "prompt_tokens": 40, + "queue_time": 0.10031125, + "time_to_first_token": 0, + "tokens": 42, + "total_time": 0.009795562 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "groq-reasoning-stream-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Solve this step by step: Elena has 3 boxes with 4 marbles each, gives away 5 marbles, then doubles what remains. Reply with just the final number.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "14", + "reasoning": "Okay, let's see. Elena has 3 boxes, each with 4 marbles. So first, I need to figure out how many marbles she has in total. If each box has 4 marbles and there are 3 boxes, that's 3 multiplied by 4. Let me write that down: 3 × 4 = 12 marbles. \n\nThen, she gives away 5 marbles. So from the initial 12, subtract 5. That would be 12 - 5. Let me calculate that: 12 minus 5 is 7. So now she has 7 marbles left.\n\nNext step is doubling what remains. So she has 7 marbles and then doubles them. Doubling means multiplying by 2. So 7 × 2. Hmm, 7 times 2 is 14. \n\nWait, let me check again to make sure I didn't make a mistake. Starting with 3 boxes ×4 marbles each: yes, 12. Then giving away 5 leaves 7. Then doubling 7 is 14. That seems right. I don't think I missed any steps here. The problem didn't mention any other actions, so the final number should be 14. I think that's it.\n", + "role": "assistant" + } + } + ], + "metadata": { + "max_completion_tokens": 512, + "model": "qwen/qwen3-32b", + "provider": "groq", + "reasoning_format": "parsed", + "stream": true, + "temperature": 0.6 + }, + "metrics": { + "completion_reasoning_tokens": 273, + "completion_time": 0.494818307, + "completion_tokens": 280, + "prompt_time": 0.002014349, + "prompt_tokens": 46, + "queue_time": 0.046694304, + "time_to_first_token": 0, + "tokens": 326, + "total_time": 0.496832656 + } + } + ], + "metadata": { + "operation": "reasoning-stream", + "testRunId": "" + } + }, + { + "name": "groq-tool-operation", + "children": [ + { + "name": "groq.chat.completions.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "Check the weather in Vienna and use the weather tool.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "City name.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_time": 0.038301771, + "completion_tokens": 11, + "prompt_time": 0.0136988, + "prompt_tokens": 246, + "queue_time": 0.091345947, + "time_to_first_token": 0, + "tokens": 257, + "total_time": 0.052000571 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "groq-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.txt b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.txt new file mode 100644 index 000000000..02dce1fca --- /dev/null +++ b/e2e/scenarios/groq-instrumentation/__snapshots__/groq-v1-wrapped.span-tree.txt @@ -0,0 +1,201 @@ +span_tree: +└── groq-instrumentation-root [task] + metadata: { + "scenario": "groq-instrumentation", + "testRunId": "" + } + ├── groq-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "OK", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_completion_tokens": 12, + │ "model": "llama-3.3-70b-versatile", + │ "provider": "groq", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_time": 0.006650543, + │ "completion_tokens": 2, + │ "prompt_time": 0.002703216, + │ "prompt_tokens": 40, + │ "queue_time": 0.109831551, + │ "time_to_first_token": 0, + │ "tokens": 42, + │ "total_time": 0.009353759 + │ } + ├── groq-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "llama-3.3-70b-versatile", + │ "provider": "groq", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_time": 0.007875603, + │ "completion_tokens": 2, + │ "prompt_time": 0.001919959, + │ "prompt_tokens": 40, + │ "queue_time": 0.10031125, + │ "time_to_first_token": 0, + │ "tokens": 42, + │ "total_time": 0.009795562 + │ } + ├── groq-reasoning-stream-operation + │ metadata: { + │ "operation": "reasoning-stream", + │ "testRunId": "" + │ } + │ └── groq.chat.completions.create [llm] + │ input: [ + │ { + │ "content": "Solve this step by step: Elena has 3 boxes with 4 marbles each, gives away 5 marbles, then doubles what remains. Reply with just the final number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "14", + │ "reasoning": "Okay, let's see. Elena has 3 boxes, each with 4 marbles. So first, I need to figure out how many marbles she has in total. If each box has 4 marbles and there are 3 boxes, that's 3 multiplied by 4. Let me write that down: 3 × 4 = 12 marbles. \n\nThen, she gives away 5 marbles. So from the initial 12, subtract 5. That would be 12 - 5. Let me calculate that: 12 minus 5 is 7. So now she has 7 marbles left.\n\nNext step is doubling what remains. So she has 7 marbles and then doubles them. Doubling means multiplying by 2. So 7 × 2. Hmm, 7 times 2 is 14. \n\nWait, let me check again to make sure I didn't make a mistake. Starting with 3 boxes ×4 marbles each: yes, 12. Then giving away 5 leaves 7. Then doubling 7 is 14. That seems right. I don't think I missed any steps here. The problem didn't mention any other actions, so the final number should be 14. I think that's it.\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_completion_tokens": 512, + │ "model": "qwen/qwen3-32b", + │ "provider": "groq", + │ "reasoning_format": "parsed", + │ "stream": true, + │ "temperature": 0.6 + │ } + │ metrics: { + │ "completion_reasoning_tokens": 273, + │ "completion_time": 0.494818307, + │ "completion_tokens": 280, + │ "prompt_time": 0.002014349, + │ "prompt_tokens": 46, + │ "queue_time": 0.046694304, + │ "time_to_first_token": 0, + │ "tokens": 326, + │ "total_time": 0.496832656 + │ } + └── groq-tool-operation + metadata: { + "operation": "tool", + "testRunId": "" + } + └── groq.chat.completions.create [llm] + input: [ + { + "content": "Check the weather in Vienna and use the weather tool.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "tool_calls", + "index": 0, + "logprobs": null, + "message": { + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ] + metadata: { + "model": "llama-3.3-70b-versatile", + "provider": "groq", + "temperature": 0, + "tool_choice": { + "function": { + "name": "get_weather" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "description": "Get the weather for a city.", + "name": "get_weather", + "parameters": { + "properties": { + "location": { + "description": "City name.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + } + metrics: { + "completion_time": 0.038301771, + "completion_tokens": 11, + "prompt_time": 0.0136988, + "prompt_tokens": 246, + "queue_time": 0.091345947, + "time_to_first_token": 0, + "tokens": 257, + "total_time": 0.052000571 + } diff --git a/e2e/scenarios/groq-instrumentation/assertions.ts b/e2e/scenarios/groq-instrumentation/assertions.ts index 7ad304ac0..45654909c 100644 --- a/e2e/scenarios/groq-instrumentation/assertions.ts +++ b/e2e/scenarios/groq-instrumentation/assertions.ts @@ -1,17 +1,12 @@ import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { REASONING_MODEL, ROOT_NAME, SCENARIO_NAME } from "./constants.mjs"; type RunGroqScenario = (harness: { @@ -39,7 +34,7 @@ function findGroqSpan( return spans.find((candidate) => candidate.output !== undefined) ?? spans[0]; } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function spanTreeEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { const chatOperation = findLatestSpan(events, "groq-chat-operation"); const streamOperation = findLatestSpan(events, "groq-stream-operation"); const reasoningStreamOperation = findLatestSpan( @@ -74,16 +69,7 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { toolOperation?.span.id, "groq.chat.completions.create", ), - ].map((event) => - summarizeWrapperContract(event!, [ - "model", - "operation", - "provider", - "reasoning_format", - "scenario", - "temperature", - ]), - ) as Json; + ].map((event) => event!); } export function defineGroqInstrumentationAssertions(options: { @@ -95,7 +81,7 @@ export function defineGroqInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -200,11 +186,8 @@ export function defineGroqInstrumentationAssertions(options: { ); }); - test("matches span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events)), - spanSnapshotPath, - ); + test("matches span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json deleted file mode 100644 index 82e500fc5..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.log-payloads.json +++ /dev/null @@ -1,427 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "huggingface-instrumentation", - "testRunId": "" - }, - "metrics": { - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "huggingface-instrumentation-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": "" - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "huggingface-chat-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "endpointUrl": "https://router.huggingface.co", - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion", - "provider": "huggingface", - "temperature": 0 - }, - "metrics": { - "completion_accepted_prediction_tokens": 0, - "completion_reasoning_tokens": 0, - "completion_rejected_prediction_tokens": 0, - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "tokens": "" - }, - "output": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ], - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "huggingface.chat_completion", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "huggingface-chat-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "endpointUrl": "https://router.huggingface.co", - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion.chunk", - "provider": "huggingface", - "temperature": 0 - }, - "metrics": { - "completion_accepted_prediction_tokens": 0, - "completion_reasoning_tokens": 0, - "completion_rejected_prediction_tokens": 0, - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream-tool-call", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "huggingface-chat-stream-tool-call-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "What is the weather in San Francisco? Call the get_current_weather tool.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "endpointUrl": "https://router.huggingface.co", - "id": "", - "max_tokens": 64, - "model": "", - "object": "chat.completion.chunk", - "provider": "featherless-ai", - "temperature": 0, - "tool_choice": "required", - "tools": [ - { - "function": { - "description": "Get the current weather for a location.", - "name": "get_current_weather", - "parameters": { - "properties": { - "location": { - "description": "City and state or city and country.", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metrics": { - "completion_accepted_prediction_tokens": 0, - "completion_reasoning_tokens": 0, - "completion_rejected_prediction_tokens": 0, - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "text-generation-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "huggingface-text-generation-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "The capital of France is", - "log_id": "g", - "metadata": { - "endpointUrl": "https://router.huggingface.co/featherless-ai/v1/completions", - "finish_reason": "length", - "max_tokens": 4, - "model": "", - "provider": "huggingface" - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "huggingface.text_generation_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "feature-extraction", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "huggingface-feature-extraction-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "Paris France", - "log_id": "g", - "metadata": { - "endpointUrl": "https://router.huggingface.co/hf-inference/models/thenlper/gte-large/pipeline/feature-extraction", - "model": "", - "provider": "huggingface" - }, - "metrics": { - "end": "", - "start": "" - }, - "output": { - "embedding_length": 1024 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "huggingface.feature_extraction", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-events.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-events.json deleted file mode 100644 index 320b17790..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-events.json +++ /dev/null @@ -1,226 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "huggingface-instrumentation" - }, - "metric_keys": [], - "name": "huggingface-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "huggingface-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "endpointUrl": "https://router.huggingface.co", - "model": "", - "provider": "huggingface" - }, - "metric_keys": [ - "completion_accepted_prediction_tokens", - "completion_reasoning_tokens", - "completion_rejected_prediction_tokens", - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "tokens" - ], - "name": "huggingface.chat_completion", - "output": [ - { - "content": "", - "finish_reason": "stop", - "index": 0, - "role": "assistant" - } - ], - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "endpointUrl": "https://router.huggingface.co", - "model": "", - "provider": "huggingface" - }, - "metric_keys": [ - "completion_accepted_prediction_tokens", - "completion_reasoning_tokens", - "completion_rejected_prediction_tokens", - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-tool-call" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "endpointUrl": "https://router.huggingface.co", - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_accepted_prediction_tokens", - "completion_reasoning_tokens", - "completion_rejected_prediction_tokens", - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - null, - null, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "text-generation-stream" - }, - "metric_keys": [], - "name": "huggingface-text-generation-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "endpointUrl": "https://router.huggingface.co/featherless-ai/v1/completions", - "finish_reason": "length", - "model": "", - "provider": "huggingface" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.text_generation_stream", - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "feature-extraction" - }, - "metric_keys": [], - "name": "huggingface-feature-extraction-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "endpointUrl": "https://router.huggingface.co/hf-inference/models/thenlper/gte-large/pipeline/feature-extraction", - "model": "", - "provider": "huggingface" - }, - "metric_keys": [], - "name": "huggingface.feature_extraction", - "output": { - "embedding_length": 1024 - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.json new file mode 100644 index 000000000..29eda3ada --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.json @@ -0,0 +1,254 @@ +{ + "span_tree": [ + { + "name": "huggingface-instrumentation-root", + "type": "task", + "children": [ + { + "name": "huggingface-chat-operation", + "children": [ + { + "name": "huggingface.chat_completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "endpointUrl": "/huggingface-router", + "id": "", + "max_tokens": 16, + "model": "llama3.1-8b", + "object": "chat.completion", + "provider": "huggingface", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 40, + "tokens": 43 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ] + }, + "metadata": { + "created": 0, + "endpointUrl": "/huggingface-router", + "id": "", + "max_tokens": 16, + "model": "llama3.1-8b", + "object": "chat.completion.chunk", + "provider": "huggingface", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 40, + "time_to_first_token": 0, + "tokens": 43 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-tool-call-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": "", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"location\": \"San Francisco\"}", + "name": "get_current_weather" + }, + "id": "", + "type": "function" + } + ] + } + } + ] + }, + "metadata": { + "created": 0, + "endpointUrl": "/huggingface-router", + "id": "", + "max_tokens": 64, + "model": "llama3.1-8b", + "object": "chat.completion.chunk", + "provider": "featherless-ai", + "temperature": 0, + "tool_choice": "required", + "tools": [ + { + "function": { + "description": "Get the current weather for a location.", + "name": "get_current_weather", + "parameters": { + "properties": { + "location": { + "description": "City and state or city and country.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 20, + "prompt_cached_tokens": 384, + "prompt_tokens": 394, + "time_to_first_token": 0, + "tokens": 414 + } + } + ], + "metadata": { + "operation": "chat-stream-tool-call", + "testRunId": "" + } + }, + { + "name": "huggingface-text-generation-stream-operation", + "children": [ + { + "name": "huggingface.text_generation_stream", + "type": "llm", + "children": [], + "input": "The capital of France is", + "output": { + "finish_reason": "length", + "generated_text": " Paris.\nThe capital" + }, + "metadata": { + "endpointUrl": "/huggingface-router/featherless-ai/v1/completions", + "finish_reason": "length", + "max_tokens": 4, + "model": "arcee-ai/Trinity-Large-Thinking", + "provider": "huggingface" + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 6, + "time_to_first_token": 0, + "tokens": 10 + } + } + ], + "metadata": { + "operation": "text-generation-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-feature-extraction-operation", + "children": [ + { + "name": "huggingface.feature_extraction", + "type": "llm", + "children": [], + "input": "Paris France", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "endpointUrl": "/huggingface-router/hf-inference/models/thenlper/gte-large/pipeline/feature-extraction", + "model": "thenlper/gte-large", + "provider": "huggingface" + } + } + ], + "metadata": { + "operation": "feature-extraction", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.txt b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.txt new file mode 100644 index 000000000..d6cdbadad --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v281.span-tree.txt @@ -0,0 +1,206 @@ +span_tree: +└── huggingface-instrumentation-root [task] + metadata: { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + ├── huggingface-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "endpointUrl": "/huggingface-router", + │ "id": "", + │ "max_tokens": 16, + │ "model": "llama3.1-8b", + │ "object": "chat.completion", + │ "provider": "huggingface", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 40, + │ "tokens": 43 + │ } + ├── huggingface-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "endpointUrl": "/huggingface-router", + │ "id": "", + │ "max_tokens": 16, + │ "model": "llama3.1-8b", + │ "object": "chat.completion.chunk", + │ "provider": "huggingface", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 40, + │ "time_to_first_token": 0, + │ "tokens": 43 + │ } + ├── huggingface-chat-stream-tool-call-operation + │ metadata: { + │ "operation": "chat-stream-tool-call", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"San Francisco\"}", + │ "name": "get_current_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "endpointUrl": "/huggingface-router", + │ "id": "", + │ "max_tokens": 64, + │ "model": "llama3.1-8b", + │ "object": "chat.completion.chunk", + │ "provider": "featherless-ai", + │ "temperature": 0, + │ "tool_choice": "required", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the current weather for a location.", + │ "name": "get_current_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "City and state or city and country.", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 20, + │ "prompt_cached_tokens": 384, + │ "prompt_tokens": 394, + │ "time_to_first_token": 0, + │ "tokens": 414 + │ } + ├── huggingface-text-generation-stream-operation + │ metadata: { + │ "operation": "text-generation-stream", + │ "testRunId": "" + │ } + │ └── huggingface.text_generation_stream [llm] + │ input: "The capital of France is" + │ output: { + │ "finish_reason": "length", + │ "generated_text": " Paris.\nThe capital" + │ } + │ metadata: { + │ "endpointUrl": "/huggingface-router/featherless-ai/v1/completions", + │ "finish_reason": "length", + │ "max_tokens": 4, + │ "model": "arcee-ai/Trinity-Large-Thinking", + │ "provider": "huggingface" + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 6, + │ "time_to_first_token": 0, + │ "tokens": 10 + │ } + └── huggingface-feature-extraction-operation + metadata: { + "operation": "feature-extraction", + "testRunId": "" + } + └── huggingface.feature_extraction [llm] + input: "Paris France" + output: { + "embedding_length": 1024 + } + metadata: { + "endpointUrl": "/huggingface-router/hf-inference/models/thenlper/gte-large/pipeline/feature-extraction", + "model": "thenlper/gte-large", + "provider": "huggingface" + } diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json deleted file mode 100644 index dab67fbd6..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.log-payloads.json +++ /dev/null @@ -1,479 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "huggingface-instrumentation", - "testRunId": "" - }, - "metrics": { - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "huggingface-instrumentation-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": "" - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "huggingface-chat-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion", - "provider": "featherless-ai", - "temperature": 0 - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "tokens": "" - }, - "output": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ], - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "huggingface.chat_completion", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "huggingface-chat-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion.chunk", - "provider": "featherless-ai", - "temperature": 0 - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream-tool-call", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "huggingface-chat-stream-tool-call-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "What is the weather in San Francisco? Call the get_current_weather tool.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 64, - "model": "", - "object": "chat.completion.chunk", - "provider": "featherless-ai", - "temperature": 0, - "tool_choice": "required", - "tools": [ - { - "function": { - "description": "Get the current weather for a location.", - "name": "get_current_weather", - "parameters": { - "properties": { - "location": { - "description": "City and state or city and country.", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "text-generation", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "huggingface-text-generation-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "The capital of France is", - "log_id": "g", - "metadata": { - "model": "", - "parameters": { - "do_sample": false, - "max_new_tokens": 4, - "return_full_text": false - }, - "provider": "featherless-ai" - }, - "metrics": { - "end": "", - "start": "" - }, - "output": { - "generated_text": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "huggingface.text_generation", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "text-generation-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "huggingface-text-generation-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "The capital of France is", - "log_id": "g", - "metadata": { - "finish_reason": "length", - "model": "", - "parameters": { - "do_sample": false, - "max_new_tokens": 4, - "return_full_text": false - }, - "provider": "featherless-ai" - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "huggingface.text_generation_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "feature-extraction", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "huggingface-feature-extraction-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "Paris France", - "log_id": "g", - "metadata": { - "model": "", - "provider": "hf-inference" - }, - "metrics": { - "end": "", - "start": "" - }, - "output": { - "embedding_length": 1024 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "huggingface.feature_extraction", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-events.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-events.json deleted file mode 100644 index f8c10443e..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-events.json +++ /dev/null @@ -1,241 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "huggingface-instrumentation" - }, - "metric_keys": [], - "name": "huggingface-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "huggingface-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "huggingface.chat_completion", - "output": [ - { - "content": "", - "finish_reason": "stop", - "index": 0, - "role": "assistant" - } - ], - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-tool-call" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "text-generation" - }, - "metric_keys": [], - "name": "huggingface-text-generation-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [], - "name": "huggingface.text_generation", - "output": { - "generated_text": "" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "text-generation-stream" - }, - "metric_keys": [], - "name": "huggingface-text-generation-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "finish_reason": "length", - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.text_generation_stream", - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "feature-extraction" - }, - "metric_keys": [], - "name": "huggingface-feature-extraction-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "hf-inference" - }, - "metric_keys": [], - "name": "huggingface.feature_extraction", - "output": { - "embedding_length": 1024 - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.json new file mode 100644 index 000000000..e568d92fd --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.json @@ -0,0 +1,258 @@ +{ + "span_tree": [ + { + "name": "huggingface-instrumentation-root", + "type": "task", + "children": [ + { + "name": "huggingface-chat-operation", + "children": [ + { + "name": "huggingface.chat_completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "max_tokens": 16, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion", + "provider": "featherless-ai", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 10, + "tokens": 12 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ] + }, + "metadata": { + "created": 0, + "id": "", + "max_tokens": 16, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion.chunk", + "provider": "featherless-ai", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 12 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-tool-call-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "I'm not capable of directly accessing real-time weather data. However, I can guide you on how to use APIs to get the current weather in San Francisco.\n\nYou can use the OpenWeatherMap API to get the current weather in San Francisco. Here's a Python example using the `requests` library:\n\n```python\n", + "role": "assistant" + } + } + ] + }, + "metadata": { + "created": 0, + "id": "", + "max_tokens": 64, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion.chunk", + "provider": "featherless-ai", + "temperature": 0, + "tool_choice": "required", + "tools": [ + { + "function": { + "description": "Get the current weather for a location.", + "name": "get_current_weather", + "parameters": { + "properties": { + "location": { + "description": "City and state or city and country.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 64, + "prompt_tokens": 20, + "time_to_first_token": 0, + "tokens": 84 + } + } + ], + "metadata": { + "operation": "chat-stream-tool-call", + "testRunId": "" + } + }, + { + "name": "huggingface-text-generation-operation", + "children": [ + { + "name": "huggingface.text_generation", + "type": "llm", + "children": [], + "input": "The capital of France is", + "output": { + "generated_text": " a city of art" + }, + "metadata": { + "model": "meta-llama/Llama-3.1-8B", + "parameters": { + "do_sample": false, + "max_new_tokens": 4, + "return_full_text": false + }, + "provider": "featherless-ai" + } + } + ], + "metadata": { + "operation": "text-generation", + "testRunId": "" + } + }, + { + "name": "huggingface-text-generation-stream-operation", + "children": [ + { + "name": "huggingface.text_generation_stream", + "type": "llm", + "children": [], + "input": "The capital of France is", + "output": { + "finish_reason": "length", + "generated_text": " one of the most" + }, + "metadata": { + "finish_reason": "length", + "model": "meta-llama/Llama-3.1-8B", + "parameters": { + "do_sample": false, + "max_new_tokens": 4, + "return_full_text": false + }, + "provider": "featherless-ai" + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "text-generation-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-feature-extraction-operation", + "children": [ + { + "name": "huggingface.feature_extraction", + "type": "llm", + "children": [], + "input": "Paris France", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "model": "thenlper/gte-large", + "provider": "hf-inference" + } + } + ], + "metadata": { + "operation": "feature-extraction", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.txt b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.txt new file mode 100644 index 000000000..7c4bcd8e5 --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v3150.span-tree.txt @@ -0,0 +1,202 @@ +span_tree: +└── huggingface-instrumentation-root [task] + metadata: { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + ├── huggingface-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 16, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion", + │ "provider": "featherless-ai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 10, + │ "tokens": 12 + │ } + ├── huggingface-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 16, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion.chunk", + │ "provider": "featherless-ai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 12 + │ } + ├── huggingface-chat-stream-tool-call-operation + │ metadata: { + │ "operation": "chat-stream-tool-call", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "length", + │ "index": 0, + │ "message": { + │ "content": "I'm not capable of directly accessing real-time weather data. However, I can guide you on how to use APIs to get the current weather in San Francisco.\n\nYou can use the OpenWeatherMap API to get the current weather in San Francisco. Here's a Python example using the `requests` library:\n\n```python\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 64, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion.chunk", + │ "provider": "featherless-ai", + │ "temperature": 0, + │ "tool_choice": "required", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the current weather for a location.", + │ "name": "get_current_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "City and state or city and country.", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 64, + │ "prompt_tokens": 20, + │ "time_to_first_token": 0, + │ "tokens": 84 + │ } + ├── huggingface-text-generation-operation + │ metadata: { + │ "operation": "text-generation", + │ "testRunId": "" + │ } + │ └── huggingface.text_generation [llm] + │ input: "The capital of France is" + │ output: { + │ "generated_text": " a city of art" + │ } + │ metadata: { + │ "model": "meta-llama/Llama-3.1-8B", + │ "parameters": { + │ "do_sample": false, + │ "max_new_tokens": 4, + │ "return_full_text": false + │ }, + │ "provider": "featherless-ai" + │ } + ├── huggingface-text-generation-stream-operation + │ metadata: { + │ "operation": "text-generation-stream", + │ "testRunId": "" + │ } + │ └── huggingface.text_generation_stream [llm] + │ input: "The capital of France is" + │ output: { + │ "finish_reason": "length", + │ "generated_text": " one of the most" + │ } + │ metadata: { + │ "finish_reason": "length", + │ "model": "meta-llama/Llama-3.1-8B", + │ "parameters": { + │ "do_sample": false, + │ "max_new_tokens": 4, + │ "return_full_text": false + │ }, + │ "provider": "featherless-ai" + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + └── huggingface-feature-extraction-operation + metadata: { + "operation": "feature-extraction", + "testRunId": "" + } + └── huggingface.feature_extraction [llm] + input: "Paris France" + output: { + "embedding_length": 1024 + } + metadata: { + "model": "thenlper/gte-large", + "provider": "hf-inference" + } diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json deleted file mode 100644 index dab67fbd6..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.log-payloads.json +++ /dev/null @@ -1,479 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "huggingface-instrumentation", - "testRunId": "" - }, - "metrics": { - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "huggingface-instrumentation-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": "" - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "huggingface-chat-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion", - "provider": "featherless-ai", - "temperature": 0 - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "tokens": "" - }, - "output": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ], - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "huggingface.chat_completion", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "huggingface-chat-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 16, - "model": "", - "object": "chat.completion.chunk", - "provider": "featherless-ai", - "temperature": 0 - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chat-stream-tool-call", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "huggingface-chat-stream-tool-call-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "What is the weather in San Francisco? Call the get_current_weather tool.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "created": 0, - "id": "", - "max_tokens": 64, - "model": "", - "object": "chat.completion.chunk", - "provider": "featherless-ai", - "temperature": 0, - "tool_choice": "required", - "tools": [ - { - "function": { - "description": "Get the current weather for a location.", - "name": "get_current_weather", - "parameters": { - "properties": { - "location": { - "description": "City and state or city and country.", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "choices": [ - { - "content": "", - "finish_reason": "", - "index": 0, - "role": "assistant" - } - ] - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "huggingface.chat_completion_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "text-generation", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "huggingface-text-generation-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "The capital of France is", - "log_id": "g", - "metadata": { - "model": "", - "parameters": { - "do_sample": false, - "max_new_tokens": 4, - "return_full_text": false - }, - "provider": "featherless-ai" - }, - "metrics": { - "end": "", - "start": "" - }, - "output": { - "generated_text": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "huggingface.text_generation", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "text-generation-stream", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "huggingface-text-generation-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "The capital of France is", - "log_id": "g", - "metadata": { - "finish_reason": "length", - "model": "", - "parameters": { - "do_sample": false, - "max_new_tokens": 4, - "return_full_text": false - }, - "provider": "featherless-ai" - }, - "metrics": { - "completion_tokens": "", - "end": "", - "prompt_tokens": "", - "start": "", - "time_to_first_token": "", - "tokens": "" - }, - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "huggingface.text_generation_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "feature-extraction", - "testRunId": "" - }, - "metrics": { - "end": "", - "start": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "huggingface-feature-extraction-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": "Paris France", - "log_id": "g", - "metadata": { - "model": "", - "provider": "hf-inference" - }, - "metrics": { - "end": "", - "start": "" - }, - "output": { - "embedding_length": 1024 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "huggingface.feature_extraction", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-events.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-events.json deleted file mode 100644 index f8c10443e..000000000 --- a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-events.json +++ /dev/null @@ -1,241 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "huggingface-instrumentation" - }, - "metric_keys": [], - "name": "huggingface-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "huggingface-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "huggingface.chat_completion", - "output": [ - { - "content": "", - "finish_reason": "stop", - "index": 0, - "role": "assistant" - } - ], - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-tool-call" - }, - "metric_keys": [], - "name": "huggingface-chat-stream-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.chat_completion_stream", - "output": null, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "text-generation" - }, - "metric_keys": [], - "name": "huggingface-text-generation-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [], - "name": "huggingface.text_generation", - "output": { - "generated_text": "" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "text-generation-stream" - }, - "metric_keys": [], - "name": "huggingface-text-generation-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "finish_reason": "length", - "model": "", - "provider": "featherless-ai" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "huggingface.text_generation_stream", - "output": { - "finish_reason": "length", - "generated_text": "" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "feature-extraction" - }, - "metric_keys": [], - "name": "huggingface-feature-extraction-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "", - "provider": "hf-inference" - }, - "metric_keys": [], - "name": "huggingface.feature_extraction", - "output": { - "embedding_length": 1024 - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.json b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.json new file mode 100644 index 000000000..c8c373a5c --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.json @@ -0,0 +1,258 @@ +{ + "span_tree": [ + { + "name": "huggingface-instrumentation-root", + "type": "task", + "children": [ + { + "name": "huggingface-chat-operation", + "children": [ + { + "name": "huggingface.chat_completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "max_tokens": 16, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion", + "provider": "featherless-ai", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 10, + "tokens": 12 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK.", + "role": "assistant" + } + } + ] + }, + "metadata": { + "created": 0, + "id": "", + "max_tokens": 16, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion.chunk", + "provider": "featherless-ai", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 12 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-chat-stream-tool-call-operation", + "children": [ + { + "name": "huggingface.chat_completion_stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + "role": "user" + } + ], + "output": { + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "I'm not capable of directly accessing real-time weather data. However, I can guide you on how to use APIs to get the current weather in San Francisco.\n\nYou can use the OpenWeatherMap API to get the current weather in San Francisco. Here's a Python example using the `requests` library:\n\n```python\n", + "role": "assistant" + } + } + ] + }, + "metadata": { + "created": 0, + "id": "", + "max_tokens": 64, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "object": "chat.completion.chunk", + "provider": "featherless-ai", + "temperature": 0, + "tool_choice": "required", + "tools": [ + { + "function": { + "description": "Get the current weather for a location.", + "name": "get_current_weather", + "parameters": { + "properties": { + "location": { + "description": "City and state or city and country.", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 64, + "prompt_tokens": 20, + "time_to_first_token": 0, + "tokens": 84 + } + } + ], + "metadata": { + "operation": "chat-stream-tool-call", + "testRunId": "" + } + }, + { + "name": "huggingface-text-generation-operation", + "children": [ + { + "name": "huggingface.text_generation", + "type": "llm", + "children": [], + "input": "The capital of France is", + "output": { + "generated_text": " the most visited city" + }, + "metadata": { + "model": "meta-llama/Llama-3.1-8B", + "parameters": { + "do_sample": false, + "max_new_tokens": 4, + "return_full_text": false + }, + "provider": "featherless-ai" + } + } + ], + "metadata": { + "operation": "text-generation", + "testRunId": "" + } + }, + { + "name": "huggingface-text-generation-stream-operation", + "children": [ + { + "name": "huggingface.text_generation_stream", + "type": "llm", + "children": [], + "input": "The capital of France is", + "output": { + "finish_reason": "length", + "generated_text": " Paris. It is" + }, + "metadata": { + "finish_reason": "length", + "model": "meta-llama/Llama-3.1-8B", + "parameters": { + "do_sample": false, + "max_new_tokens": 4, + "return_full_text": false + }, + "provider": "featherless-ai" + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "text-generation-stream", + "testRunId": "" + } + }, + { + "name": "huggingface-feature-extraction-operation", + "children": [ + { + "name": "huggingface.feature_extraction", + "type": "llm", + "children": [], + "input": "Paris France", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "model": "thenlper/gte-large", + "provider": "hf-inference" + } + } + ], + "metadata": { + "operation": "feature-extraction", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.txt b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.txt new file mode 100644 index 000000000..24243e3e8 --- /dev/null +++ b/e2e/scenarios/huggingface-instrumentation/__snapshots__/huggingface-v41315.span-tree.txt @@ -0,0 +1,202 @@ +span_tree: +└── huggingface-instrumentation-root [task] + metadata: { + "scenario": "huggingface-instrumentation", + "testRunId": "" + } + ├── huggingface-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 16, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion", + │ "provider": "featherless-ai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 10, + │ "tokens": 12 + │ } + ├── huggingface-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 16, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion.chunk", + │ "provider": "featherless-ai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 12 + │ } + ├── huggingface-chat-stream-tool-call-operation + │ metadata: { + │ "operation": "chat-stream-tool-call", + │ "testRunId": "" + │ } + │ └── huggingface.chat_completion_stream [llm] + │ input: [ + │ { + │ "content": "What is the weather in San Francisco? Call the get_current_weather tool.", + │ "role": "user" + │ } + │ ] + │ output: { + │ "choices": [ + │ { + │ "finish_reason": "length", + │ "index": 0, + │ "message": { + │ "content": "I'm not capable of directly accessing real-time weather data. However, I can guide you on how to use APIs to get the current weather in San Francisco.\n\nYou can use the OpenWeatherMap API to get the current weather in San Francisco. Here's a Python example using the `requests` library:\n\n```python\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ } + │ metadata: { + │ "created": 0, + │ "id": "", + │ "max_tokens": 64, + │ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + │ "object": "chat.completion.chunk", + │ "provider": "featherless-ai", + │ "temperature": 0, + │ "tool_choice": "required", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the current weather for a location.", + │ "name": "get_current_weather", + │ "parameters": { + │ "properties": { + │ "location": { + │ "description": "City and state or city and country.", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 64, + │ "prompt_tokens": 20, + │ "time_to_first_token": 0, + │ "tokens": 84 + │ } + ├── huggingface-text-generation-operation + │ metadata: { + │ "operation": "text-generation", + │ "testRunId": "" + │ } + │ └── huggingface.text_generation [llm] + │ input: "The capital of France is" + │ output: { + │ "generated_text": " the most visited city" + │ } + │ metadata: { + │ "model": "meta-llama/Llama-3.1-8B", + │ "parameters": { + │ "do_sample": false, + │ "max_new_tokens": 4, + │ "return_full_text": false + │ }, + │ "provider": "featherless-ai" + │ } + ├── huggingface-text-generation-stream-operation + │ metadata: { + │ "operation": "text-generation-stream", + │ "testRunId": "" + │ } + │ └── huggingface.text_generation_stream [llm] + │ input: "The capital of France is" + │ output: { + │ "finish_reason": "length", + │ "generated_text": " Paris. It is" + │ } + │ metadata: { + │ "finish_reason": "length", + │ "model": "meta-llama/Llama-3.1-8B", + │ "parameters": { + │ "do_sample": false, + │ "max_new_tokens": 4, + │ "return_full_text": false + │ }, + │ "provider": "featherless-ai" + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 9 + │ } + └── huggingface-feature-extraction-operation + metadata: { + "operation": "feature-extraction", + "testRunId": "" + } + └── huggingface.feature_extraction [llm] + input: "Paris France" + output: { + "embedding_length": 1024 + } + metadata: { + "model": "thenlper/gte-large", + "provider": "hf-inference" + } diff --git a/e2e/scenarios/huggingface-instrumentation/assertions.ts b/e2e/scenarios/huggingface-instrumentation/assertions.ts index 3bbaed9fe..6fba78ec8 100644 --- a/e2e/scenarios/huggingface-instrumentation/assertions.ts +++ b/e2e/scenarios/huggingface-instrumentation/assertions.ts @@ -1,11 +1,7 @@ import { beforeAll, describe, expect, test } from "vitest"; import { type Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, @@ -15,9 +11,11 @@ import { findLatestSpan, } from "../../helpers/trace-selectors"; import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "../../helpers/wrapper-contract"; + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, + type SpanTreeFields, +} from "../../helpers/span-tree"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -120,36 +118,6 @@ function summarizeTextGenerationOutput(output: Json | undefined): Json { } satisfies Json; } -function summarizeProviderSpan(event: CapturedLogEvent): Json { - const summary = summarizeWrapperContract(event, [ - "dimensions", - "endpointUrl", - "finish_reason", - "model", - "operation", - "provider", - "scenario", - ]) as Record; - - switch (event.span.name) { - case "huggingface.chat_completion": - case "huggingface.chat_completion_stream": - summary.output = summarizeChatOutput(event.output as Json); - break; - case "huggingface.text_generation": - case "huggingface.text_generation_stream": - summary.output = summarizeTextGenerationOutput(event.output as Json); - break; - case "huggingface.feature_extraction": - summary.output = (event.output as Json) ?? null; - break; - default: - break; - } - - return normalizeEndpointUrls(summary); -} - function normalizeEndpointUrl(value: string): string { try { const url = new URL(value); @@ -241,24 +209,6 @@ function normalizeModelNames(value: Json): Json { return normalized; } -function normalizePayloadOutput(row: Json): Json { - if (!isRecord(row)) { - return row; - } - - const normalized = - "output" in row - ? { - ...row, - output: normalizeLoggedOutput(row.output, { - normalizeFinishReason: true, - omitToolCalls: true, - }), - } - : row; - return normalizeEndpointUrls(normalizeModelNames(normalized)); -} - function normalizeLoggedOutput( output: Json, options?: { @@ -288,7 +238,26 @@ function normalizeLoggedOutput( return output; } -function buildSpanSummary(events: CapturedLogEvent[]): Json { +function snapshotFields(event: CapturedLogEvent): SpanTreeFields { + const fields = spanTreeFields(event); + const output = + event.output === undefined + ? undefined + : normalizeLoggedOutput(event.output as Json, { + normalizeFinishReason: true, + omitToolCalls: true, + }); + + return normalizeEndpointUrls( + normalizeModelNames({ + ...fields, + metrics: normalizeMetrics(fields.metrics as Json), + ...(output === undefined ? {} : { output }), + } as Json), + ) as SpanTreeFields; +} + +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { const root = findLatestSpan(events, ROOT_NAME); const chatOperation = findLatestSpan(events, "huggingface-chat-operation"); const chatStreamOperation = findLatestSpan( @@ -312,81 +281,61 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json { "huggingface-feature-extraction-operation", ); - return normalizeModelNames([ - root ? summarizeWrapperContract(root, ["scenario"]) : null, + const relevantEvents = [ + root, + chatOperation, chatOperation - ? summarizeWrapperContract(chatOperation, ["operation"]) - : null, - chatOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.chat_completion", - chatOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.chat_completion", + chatOperation.span.id, ) - : null, + : undefined, + chatStreamOperation, chatStreamOperation - ? summarizeWrapperContract(chatStreamOperation, ["operation"]) - : null, - chatStreamOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.chat_completion_stream", - chatStreamOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.chat_completion_stream", + chatStreamOperation.span.id, ) - : null, + : undefined, + chatStreamToolCallOperation, chatStreamToolCallOperation - ? summarizeWrapperContract(chatStreamToolCallOperation, ["operation"]) - : null, - chatStreamToolCallOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.chat_completion_stream", - chatStreamToolCallOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.chat_completion_stream", + chatStreamToolCallOperation.span.id, ) - : null, - textGenerationOperation - ? summarizeWrapperContract(textGenerationOperation, ["operation"]) - : null, + : undefined, + textGenerationOperation, textGenerationOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.text_generation", - textGenerationOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.text_generation", + textGenerationOperation.span.id, ) - : null, + : undefined, + textGenerationStreamOperation, textGenerationStreamOperation - ? summarizeWrapperContract(textGenerationStreamOperation, ["operation"]) - : null, - textGenerationStreamOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.text_generation_stream", - textGenerationStreamOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.text_generation_stream", + textGenerationStreamOperation.span.id, ) - : null, - featureExtractionOperation - ? summarizeWrapperContract(featureExtractionOperation, ["operation"]) - : null, + : undefined, + featureExtractionOperation, featureExtractionOperation - ? summarizeProviderSpan( - findLatestChildSpan( - events, - "huggingface.feature_extraction", - featureExtractionOperation.span.id, - )!, + ? findLatestChildSpan( + events, + "huggingface.feature_extraction", + featureExtractionOperation.span.id, ) - : null, - ] satisfies Json); + : undefined, + ]; + + return relevantEvents.flatMap((event) => + event ? [{ event, fields: snapshotFields(event) }] : [], + ); } export function defineHuggingFaceInstrumentationAssertions(options: { @@ -398,26 +347,16 @@ export function defineHuggingFaceInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); describe(options.name, () => { let events: CapturedLogEvent[] = []; - let payloadRows: Json = []; beforeAll(async () => { await withScenarioHarness(async (harness) => { await options.runScenario(harness); events = harness.events(); - - const root = findLatestSpan(events, ROOT_NAME); - payloadRows = payloadRowsForRootSpan(harness.payloads(), root?.span.id) - .map((row) => normalizePayloadOutput(normalizeMetrics(row as Json))) - .filter((row) => row !== null); }); }, options.timeoutMs); @@ -438,21 +377,7 @@ export function defineHuggingFaceInstrumentationAssertions(options: { "matches the span contract snapshot", { timeout: options.timeoutMs }, async ({ expect }) => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events)), - spanSnapshotPath, - ); - }, - ); - - test( - "matches the log payload snapshot", - { timeout: options.timeoutMs }, - async ({ expect }) => { - await matchFileSnapshot( - formatJsonFileSnapshot(payloadRows), - payloadSnapshotPath, - ); + await matchSpanTreeSnapshot(events, spanSnapshotPath); }, ); diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json deleted file mode 100644 index d0624284c..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ /dev/null @@ -1,564 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "length", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json deleted file mode 100644 index 5242f88b3..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ /dev/null @@ -1,515 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "name": "mistral-chat-thinking-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderateChat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.json new file mode 100644 index 000000000..abc67e461 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.json @@ -0,0 +1,748 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - The first sister is currently **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Let the older brother's age be \\(B\\).\n - \\(11 = \\frac{1}{2}B\\) → \\(B = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children (3 sisters and 1", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 256, + "prompt_tokens": 83, + "time_to_first_token": 0, + "tokens": 339 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-thinking-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "4", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 1024, + "model": "magistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 28, + "time_to_first_token": 0, + "tokens": 30 + } + } + ], + "metadata": { + "operation": "chat-stream-thinking", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_tokens": 91, + "time_to_first_token": 0, + "tokens": 104 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_tokens": 125, + "time_to_first_token": 0, + "tokens": 147 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 1 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_tokens": 219, + "time_to_first_token": 0, + "tokens": 253 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n type:", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 9, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_time_in_city" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 14, + "prompt_tokens": 107, + "time_to_first_token": 0, + "tokens": 121 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Understood.", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-operation", + "children": [ + { + "name": "mistral.classifiers.moderate", + "type": "llm", + "children": [], + "input": "A short and harmless moderation fixture.", + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.00806199200451374, + "dangerous": 0.005911068990826607, + "financial": 0.00011591934162424877, + "hate_and_discrimination": 0.0023231625091284513, + "health": 0.0003799845289904624, + "jailbreaking": 0.018689308315515518, + "law": 0.00020342697098385543, + "pii": 0.001098731067031622, + "selfharm": 0.0006361911655403674, + "sexual": 0.0031236486975103617, + "violence_and_threats": 0.004609572701156139 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-chat-operation", + "children": [ + { + "name": "mistral.classifiers.moderateChat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ], + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.txt new file mode 100644 index 000000000..de3b9e25a --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-tree.txt @@ -0,0 +1,628 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - The first sister is currently **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Let the older brother's age be \\(B\\).\n - \\(11 = \\frac{1}{2}B\\) → \\(B = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children (3 sisters and 1", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 256, + │ "prompt_tokens": 83, + │ "time_to_first_token": 0, + │ "tokens": 339 + │ } + ├── mistral-chat-thinking-stream-operation + │ metadata: { + │ "operation": "chat-stream-thinking", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "4", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 1024, + │ "model": "magistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 28, + │ "time_to_first_token": 0, + │ "tokens": 30 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_tokens": 91, + │ │ "time_to_first_token": 0, + │ │ "tokens": 104 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_tokens": 125, + │ │ "time_to_first_token": 0, + │ │ "tokens": 147 + │ │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "index": 0 + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "", + │ "index": 1 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_tokens": 219, + │ "time_to_first_token": 0, + │ "tokens": 253 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n type:", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_tokens": 9, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"city\": \"Vienna\"}", + │ "name": "get_time_in_city" + │ }, + │ "id": "", + │ "index": 0 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "prompt_tokens": 107, + │ "time_to_first_token": 0, + │ "tokens": 121 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Understood.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-embeddings-operation + │ metadata: { + │ "operation": "embeddings-create", + │ "testRunId": "" + │ } + │ └── mistral.embeddings.create [llm] + │ input: "braintrust mistral instrumentation" + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "id": "", + │ "model": "mistral-embed", + │ "object": "list", + │ "provider": "mistral" + │ } + │ metrics: { + │ "completion_tokens": 0, + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } + ├── mistral-classifiers-moderate-operation + │ metadata: { + │ "operation": "classifiers-moderate", + │ "testRunId": "" + │ } + │ └── mistral.classifiers.moderate [llm] + │ input: "A short and harmless moderation fixture." + │ output: [ + │ { + │ "categories": { + │ "criminal": false, + │ "dangerous": false, + │ "financial": false, + │ "hate_and_discrimination": false, + │ "health": false, + │ "jailbreaking": false, + │ "law": false, + │ "pii": false, + │ "selfharm": false, + │ "sexual": false, + │ "violence_and_threats": false + │ }, + │ "categoryScores": { + │ "criminal": 0.00806199200451374, + │ "dangerous": 0.005911068990826607, + │ "financial": 0.00011591934162424877, + │ "hate_and_discrimination": 0.0023231625091284513, + │ "health": 0.0003799845289904624, + │ "jailbreaking": 0.018689308315515518, + │ "law": 0.00020342697098385543, + │ "pii": 0.001098731067031622, + │ "selfharm": 0.0006361911655403674, + │ "sexual": 0.0031236486975103617, + │ "violence_and_threats": 0.004609572701156139 + │ } + │ } + │ ] + │ metadata: { + │ "id": "", + │ "model": "mistral-moderation-2603", + │ "provider": "mistral" + │ } + └── mistral-classifiers-moderate-chat-operation + metadata: { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + └── mistral.classifiers.moderateChat [llm] + input: [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ] + output: [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ] + metadata: { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json deleted file mode 100644 index d0624284c..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ /dev/null @@ -1,564 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "length", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json deleted file mode 100644 index 5242f88b3..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ /dev/null @@ -1,515 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "name": "mistral-chat-thinking-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderateChat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.json new file mode 100644 index 000000000..f0fe966b3 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.json @@ -0,0 +1,748 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister (Sister 1):**\n - Current age: 4 years old.\n\n2. **Next Year (Sister 2):**\n - Next year, Sister 2 will be twice as old as Sister 1.\n - Sister 1's age next year: 4 + 1 = 5 years.\n - Therefore, Sister 2's age next year: 2 × 5 = 10 years.\n - **Current age of Sister 2:** 10 - 1 = 9 years.\n\n3. **Third Sister (Sister 3):**\n - Sister 3 is two years older than Sister 2.\n - **Current age of Sister 3:** 9 + 2 = 11 years.\n\n4. **Older Brother (John):**\n - Sister 3 is half the age of her older brother.\n - Therefore, John's age: 11 × 2 = 22 years.\n\n**Final Answer:**\n22", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 229, + "prompt_tokens": 83, + "time_to_first_token": 0, + "tokens": 312 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-thinking-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "4", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 1024, + "model": "magistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 28, + "time_to_first_token": 0, + "tokens": 30 + } + } + ], + "metadata": { + "operation": "chat-stream-thinking", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_tokens": 91, + "time_to_first_token": 0, + "tokens": 104 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_tokens": 125, + "time_to_first_token": 0, + "tokens": 147 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 1 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_tokens": 219, + "time_to_first_token": 0, + "tokens": 253 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n type:", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 9, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_time_in_city" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 14, + "prompt_tokens": 107, + "time_to_first_token": 0, + "tokens": 121 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "agent stream", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-operation", + "children": [ + { + "name": "mistral.classifiers.moderate", + "type": "llm", + "children": [], + "input": "A short and harmless moderation fixture.", + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.00806199200451374, + "dangerous": 0.005911068990826607, + "financial": 0.00011591934162424877, + "hate_and_discrimination": 0.0023231625091284513, + "health": 0.0003799845289904624, + "jailbreaking": 0.018689308315515518, + "law": 0.00020342697098385543, + "pii": 0.001098731067031622, + "selfharm": 0.0006361911655403674, + "sexual": 0.0031236486975103617, + "violence_and_threats": 0.004609572701156139 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-chat-operation", + "children": [ + { + "name": "mistral.classifiers.moderateChat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ], + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.txt new file mode 100644 index 000000000..83f5106e0 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-tree.txt @@ -0,0 +1,628 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister (Sister 1):**\n - Current age: 4 years old.\n\n2. **Next Year (Sister 2):**\n - Next year, Sister 2 will be twice as old as Sister 1.\n - Sister 1's age next year: 4 + 1 = 5 years.\n - Therefore, Sister 2's age next year: 2 × 5 = 10 years.\n - **Current age of Sister 2:** 10 - 1 = 9 years.\n\n3. **Third Sister (Sister 3):**\n - Sister 3 is two years older than Sister 2.\n - **Current age of Sister 3:** 9 + 2 = 11 years.\n\n4. **Older Brother (John):**\n - Sister 3 is half the age of her older brother.\n - Therefore, John's age: 11 × 2 = 22 years.\n\n**Final Answer:**\n22", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 229, + │ "prompt_tokens": 83, + │ "time_to_first_token": 0, + │ "tokens": 312 + │ } + ├── mistral-chat-thinking-stream-operation + │ metadata: { + │ "operation": "chat-stream-thinking", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "4", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 1024, + │ "model": "magistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 28, + │ "time_to_first_token": 0, + │ "tokens": 30 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_tokens": 91, + │ │ "time_to_first_token": 0, + │ │ "tokens": 104 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_tokens": 125, + │ │ "time_to_first_token": 0, + │ │ "tokens": 147 + │ │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "index": 0 + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "", + │ "index": 1 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_tokens": 219, + │ "time_to_first_token": 0, + │ "tokens": 253 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n type:", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_tokens": 9, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"city\": \"Vienna\"}", + │ "name": "get_time_in_city" + │ }, + │ "id": "", + │ "index": 0 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "prompt_tokens": 107, + │ "time_to_first_token": 0, + │ "tokens": 121 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "agent stream", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-embeddings-operation + │ metadata: { + │ "operation": "embeddings-create", + │ "testRunId": "" + │ } + │ └── mistral.embeddings.create [llm] + │ input: "braintrust mistral instrumentation" + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "id": "", + │ "model": "mistral-embed", + │ "object": "list", + │ "provider": "mistral" + │ } + │ metrics: { + │ "completion_tokens": 0, + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } + ├── mistral-classifiers-moderate-operation + │ metadata: { + │ "operation": "classifiers-moderate", + │ "testRunId": "" + │ } + │ └── mistral.classifiers.moderate [llm] + │ input: "A short and harmless moderation fixture." + │ output: [ + │ { + │ "categories": { + │ "criminal": false, + │ "dangerous": false, + │ "financial": false, + │ "hate_and_discrimination": false, + │ "health": false, + │ "jailbreaking": false, + │ "law": false, + │ "pii": false, + │ "selfharm": false, + │ "sexual": false, + │ "violence_and_threats": false + │ }, + │ "categoryScores": { + │ "criminal": 0.00806199200451374, + │ "dangerous": 0.005911068990826607, + │ "financial": 0.00011591934162424877, + │ "hate_and_discrimination": 0.0023231625091284513, + │ "health": 0.0003799845289904624, + │ "jailbreaking": 0.018689308315515518, + │ "law": 0.00020342697098385543, + │ "pii": 0.001098731067031622, + │ "selfharm": 0.0006361911655403674, + │ "sexual": 0.0031236486975103617, + │ "violence_and_threats": 0.004609572701156139 + │ } + │ } + │ ] + │ metadata: { + │ "id": "", + │ "model": "mistral-moderation-2603", + │ "provider": "mistral" + │ } + └── mistral-classifiers-moderate-chat-operation + metadata: { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + └── mistral.classifiers.moderateChat [llm] + input: [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ] + output: [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ] + metadata: { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json deleted file mode 100644 index d0624284c..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ /dev/null @@ -1,564 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "length", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json deleted file mode 100644 index 5242f88b3..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ /dev/null @@ -1,515 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "name": "mistral-chat-thinking-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderateChat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.json new file mode 100644 index 000000000..f706c301e --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.json @@ -0,0 +1,748 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - The first sister is currently **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Therefore, the brother's age: \\(11 \\times 2 = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children (3 sisters and 1 brother).\n - Since the brother is 22 years old,", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 256, + "prompt_tokens": 83, + "time_to_first_token": 0, + "tokens": 339 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-thinking-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "4", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 1024, + "model": "magistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 28, + "time_to_first_token": 0, + "tokens": 30 + } + } + ], + "metadata": { + "operation": "chat-stream-thinking", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_tokens": 91, + "time_to_first_token": 0, + "tokens": 104 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_tokens": 125, + "time_to_first_token": 0, + "tokens": 147 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 1 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_tokens": 219, + "time_to_first_token": 0, + "tokens": 253 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n type:", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 9, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_time_in_city" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 14, + "prompt_tokens": 107, + "time_to_first_token": 0, + "tokens": 121 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Understood.", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-operation", + "children": [ + { + "name": "mistral.classifiers.moderate", + "type": "llm", + "children": [], + "input": "A short and harmless moderation fixture.", + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.00806199200451374, + "dangerous": 0.005911068990826607, + "financial": 0.00011591934162424877, + "hate_and_discrimination": 0.0023231625091284513, + "health": 0.0003799845289904624, + "jailbreaking": 0.018689308315515518, + "law": 0.00020342697098385543, + "pii": 0.001098731067031622, + "selfharm": 0.0006361911655403674, + "sexual": 0.0031236486975103617, + "violence_and_threats": 0.004609572701156139 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-chat-operation", + "children": [ + { + "name": "mistral.classifiers.moderateChat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ], + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.txt new file mode 100644 index 000000000..86126fd44 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-tree.txt @@ -0,0 +1,628 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - The first sister is currently **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Therefore, the brother's age: \\(11 \\times 2 = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children (3 sisters and 1 brother).\n - Since the brother is 22 years old,", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 256, + │ "prompt_tokens": 83, + │ "time_to_first_token": 0, + │ "tokens": 339 + │ } + ├── mistral-chat-thinking-stream-operation + │ metadata: { + │ "operation": "chat-stream-thinking", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "4", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 1024, + │ "model": "magistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 28, + │ "time_to_first_token": 0, + │ "tokens": 30 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_tokens": 91, + │ │ "time_to_first_token": 0, + │ │ "tokens": 104 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_tokens": 125, + │ │ "time_to_first_token": 0, + │ │ "tokens": 147 + │ │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "index": 0 + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "", + │ "index": 1 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_tokens": 219, + │ "time_to_first_token": 0, + │ "tokens": 253 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n type:", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_tokens": 9, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"city\": \"Vienna\"}", + │ "name": "get_time_in_city" + │ }, + │ "id": "", + │ "index": 0 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "prompt_tokens": 107, + │ "time_to_first_token": 0, + │ "tokens": 121 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Understood.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-embeddings-operation + │ metadata: { + │ "operation": "embeddings-create", + │ "testRunId": "" + │ } + │ └── mistral.embeddings.create [llm] + │ input: "braintrust mistral instrumentation" + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "id": "", + │ "model": "mistral-embed", + │ "object": "list", + │ "provider": "mistral" + │ } + │ metrics: { + │ "completion_tokens": 0, + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } + ├── mistral-classifiers-moderate-operation + │ metadata: { + │ "operation": "classifiers-moderate", + │ "testRunId": "" + │ } + │ └── mistral.classifiers.moderate [llm] + │ input: "A short and harmless moderation fixture." + │ output: [ + │ { + │ "categories": { + │ "criminal": false, + │ "dangerous": false, + │ "financial": false, + │ "hate_and_discrimination": false, + │ "health": false, + │ "jailbreaking": false, + │ "law": false, + │ "pii": false, + │ "selfharm": false, + │ "sexual": false, + │ "violence_and_threats": false + │ }, + │ "categoryScores": { + │ "criminal": 0.00806199200451374, + │ "dangerous": 0.005911068990826607, + │ "financial": 0.00011591934162424877, + │ "hate_and_discrimination": 0.0023231625091284513, + │ "health": 0.0003799845289904624, + │ "jailbreaking": 0.018689308315515518, + │ "law": 0.00020342697098385543, + │ "pii": 0.001098731067031622, + │ "selfharm": 0.0006361911655403674, + │ "sexual": 0.0031236486975103617, + │ "violence_and_threats": 0.004609572701156139 + │ } + │ } + │ ] + │ metadata: { + │ "id": "", + │ "model": "mistral-moderation-2603", + │ "provider": "mistral" + │ } + └── mistral-classifiers-moderate-chat-operation + metadata: { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + └── mistral.classifiers.moderateChat [llm] + input: [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ] + output: [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ] + metadata: { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json deleted file mode 100644 index 459b764d3..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ /dev/null @@ -1,451 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "time_to_first_token" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json deleted file mode 100644 index 99c6533df..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ /dev/null @@ -1,405 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "time_to_first_token" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "time_to_first_token" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.json new file mode 100644 index 000000000..bed0f44ce --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.json @@ -0,0 +1,666 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": null, + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is 4 years old.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: 4 + 1 = 5 years.\n - Therefore, second sister's age next year: 2 × 5 = 10 years.\n - Current age of the second sister: 10 - 1 = 9 years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - Current age of the third sister: 9 + 2 = 11 years.\n\n4. **John's Age:**\n - The third sister is half the age of her older brother (John).\n - Therefore, John's age: 11 × 2", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 48, + "model": "mistral-small-2506", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "error": "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "" + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_tokens": 55, + "time_to_first_token": 0, + "tokens": 68 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 48, + "model": "mistral-small-2506", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "error": "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "" + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_tokens": 58, + "time_to_first_token": 0, + "tokens": 80 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "metadata": { + "maxTokens": 96, + "model": "mistral-small-2506", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "error": "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "" + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "" + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_tokens": 116, + "time_to_first_token": 0, + "tokens": 150 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": null, + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n ", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "metadata": { + "agentId": "", + "maxTokens": 32, + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "error": "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + }, + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{}", + "name": "get_time_in_city" + }, + "id": "" + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 71, + "time_to_first_token": 0, + "tokens": 79 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": null, + "index": 0, + "message": { + "content": "Understood.", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.txt new file mode 100644 index 000000000..36814c10c --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-tree.txt @@ -0,0 +1,554 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": null, + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is 4 years old.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: 4 + 1 = 5 years.\n - Therefore, second sister's age next year: 2 × 5 = 10 years.\n - Current age of the second sister: 10 - 1 = 9 years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - Current age of the third sister: 9 + 2 = 11 years.\n\n4. **John's Age:**\n - The third sister is half the age of her older brother (John).\n - Therefore, John's age: 11 × 2", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ error: "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"city\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "" + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_tokens": 55, + │ │ "time_to_first_token": 0, + │ │ "tokens": 68 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ error: "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "" + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_tokens": 58, + │ │ "time_to_first_token": 0, + │ │ "tokens": 80 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "You must return only tool calls and no plain text.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "maxTokens": 96, + │ │ "model": "mistral-small-2506", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ error: "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "" + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_tokens": 116, + │ "time_to_first_token": 0, + │ "tokens": 150 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": null, + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n ", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.agents.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ metadata: { + │ │ "agentId": "", + │ │ "maxTokens": 32, + │ │ "provider": "mistral", + │ │ "responseFormat": { + │ │ "type": "text" + │ │ }, + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ error: "Input validation failed: TypeError: Cannot read properties of undefined (reading '_zod')" + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{}", + │ "name": "get_time_in_city" + │ }, + │ "id": "" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 71, + │ "time_to_first_token": 0, + │ "tokens": 79 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": null, + │ "index": 0, + │ "message": { + │ "content": "Understood.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + └── mistral-embeddings-operation + metadata: { + "operation": "embeddings-create", + "testRunId": "" + } + └── mistral.embeddings.create [llm] + input: "braintrust mistral instrumentation" + output: { + "embedding_length": 1024 + } + metadata: { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + } + metrics: { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json deleted file mode 100644 index d0624284c..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ /dev/null @@ -1,564 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "length", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json deleted file mode 100644 index 5242f88b3..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ /dev/null @@ -1,515 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "name": "mistral-chat-thinking-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderateChat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.json new file mode 100644 index 000000000..031c25891 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.json @@ -0,0 +1,748 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **John's Age:**\n - The third sister is half the age of her older brother (John).\n - Therefore, John's age: \\(11 \\times 2 = 22\\) years.\n\n**Final Answer:**\n\\[\n\\boxed{22}\n\\]", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 234, + "prompt_tokens": 83, + "time_to_first_token": 0, + "tokens": 317 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-thinking-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "4", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 1024, + "model": "magistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_tokens": 28, + "time_to_first_token": 0, + "tokens": 30 + } + } + ], + "metadata": { + "operation": "chat-stream-thinking", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_tokens": 91, + "time_to_first_token": 0, + "tokens": 104 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_tokens": 125, + "time_to_first_token": 0, + "tokens": 147 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 1 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_tokens": 219, + "time_to_first_token": 0, + "tokens": 253 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n type:", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "prompt_tokens": 9, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_time_in_city" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 14, + "prompt_tokens": 107, + "time_to_first_token": 0, + "tokens": 121 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Understood.", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-operation", + "children": [ + { + "name": "mistral.classifiers.moderate", + "type": "llm", + "children": [], + "input": "A short and harmless moderation fixture.", + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.00806199200451374, + "dangerous": 0.005911068990826607, + "financial": 0.00011591934162424877, + "hate_and_discrimination": 0.0023231625091284513, + "health": 0.0003799845289904624, + "jailbreaking": 0.018689308315515518, + "law": 0.00020342697098385543, + "pii": 0.001098731067031622, + "selfharm": 0.0006361911655403674, + "sexual": 0.0031236486975103617, + "violence_and_threats": 0.004609572701156139 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-chat-operation", + "children": [ + { + "name": "mistral.classifiers.moderateChat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ], + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.txt new file mode 100644 index 000000000..910e9acc4 --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-tree.txt @@ -0,0 +1,628 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **John's Age:**\n - The third sister is half the age of her older brother (John).\n - Therefore, John's age: \\(11 \\times 2 = 22\\) years.\n\n**Final Answer:**\n\\[\n\\boxed{22}\n\\]", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 234, + │ "prompt_tokens": 83, + │ "time_to_first_token": 0, + │ "tokens": 317 + │ } + ├── mistral-chat-thinking-stream-operation + │ metadata: { + │ "operation": "chat-stream-thinking", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "4", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 1024, + │ "model": "magistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_tokens": 28, + │ "time_to_first_token": 0, + │ "tokens": 30 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_tokens": 91, + │ │ "time_to_first_token": 0, + │ │ "tokens": 104 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_tokens": 125, + │ │ "time_to_first_token": 0, + │ │ "tokens": 147 + │ │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "index": 0 + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "", + │ "index": 1 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_tokens": 219, + │ "time_to_first_token": 0, + │ "tokens": 253 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n type:", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_tokens": 9, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"city\": \"Vienna\"}", + │ "name": "get_time_in_city" + │ }, + │ "id": "", + │ "index": 0 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "prompt_tokens": 107, + │ "time_to_first_token": 0, + │ "tokens": 121 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Understood.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-embeddings-operation + │ metadata: { + │ "operation": "embeddings-create", + │ "testRunId": "" + │ } + │ └── mistral.embeddings.create [llm] + │ input: "braintrust mistral instrumentation" + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "id": "", + │ "model": "mistral-embed", + │ "object": "list", + │ "provider": "mistral" + │ } + │ metrics: { + │ "completion_tokens": 0, + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } + ├── mistral-classifiers-moderate-operation + │ metadata: { + │ "operation": "classifiers-moderate", + │ "testRunId": "" + │ } + │ └── mistral.classifiers.moderate [llm] + │ input: "A short and harmless moderation fixture." + │ output: [ + │ { + │ "categories": { + │ "criminal": false, + │ "dangerous": false, + │ "financial": false, + │ "hate_and_discrimination": false, + │ "health": false, + │ "jailbreaking": false, + │ "law": false, + │ "pii": false, + │ "selfharm": false, + │ "sexual": false, + │ "violence_and_threats": false + │ }, + │ "categoryScores": { + │ "criminal": 0.00806199200451374, + │ "dangerous": 0.005911068990826607, + │ "financial": 0.00011591934162424877, + │ "hate_and_discrimination": 0.0023231625091284513, + │ "health": 0.0003799845289904624, + │ "jailbreaking": 0.018689308315515518, + │ "law": 0.00020342697098385543, + │ "pii": 0.001098731067031622, + │ "selfharm": 0.0006361911655403674, + │ "sexual": 0.0031236486975103617, + │ "violence_and_threats": 0.004609572701156139 + │ } + │ } + │ ] + │ metadata: { + │ "id": "", + │ "model": "mistral-moderation-2603", + │ "provider": "mistral" + │ } + └── mistral-classifiers-moderate-chat-operation + metadata: { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + └── mistral.classifiers.moderateChat [llm] + input: [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ] + output: [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ] + metadata: { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json deleted file mode 100644 index d57f77c09..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ /dev/null @@ -1,576 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": null, - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 2, - "roles": [ - "system", - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 2, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "length", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "tool_calls", - "has_content": false, - "role": "assistant", - "tool_call_count": 1, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "output": { - "choice_count": 1, - "finish_reason": "stop", - "has_content": true, - "role": "assistant", - "tool_call_count": 0, - "type": "choices" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "output": { - "embedding_length": "", - "type": "embedding" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": "", - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - }, - { - "has_input": false, - "has_output": false, - "input": null, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "output": null, - "span_id": "" - }, - { - "has_input": true, - "has_output": true, - "input": { - "item_count": 1, - "roles": [ - "user" - ], - "type": "messages" - }, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "output": { - "choice_count": 1, - "finish_reason": null, - "type": "array" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json deleted file mode 100644 index 842dff64e..000000000 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ /dev/null @@ -1,527 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "mistral-instrumentation" - }, - "metric_keys": [], - "name": "mistral-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-complete" - }, - "metric_keys": [], - "name": "mistral-chat-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "mistral-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-reasoning" - }, - "metric_keys": [], - "name": "mistral-chat-reasoning-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-latest", - "provider": "mistral", - "reasoning_effort": "high" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream-thinking" - }, - "metric_keys": [], - "name": "mistral-chat-thinking-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "magistral-small-latest", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-tool-call" - }, - "metric_keys": [], - "name": "mistral-chat-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.chat.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-complete" - }, - "metric_keys": [], - "name": "mistral-fim-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "fim-stream" - }, - "metric_keys": [], - "name": "mistral-fim-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "codestral-2508", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.fim.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-complete" - }, - "metric_keys": [], - "name": "mistral-agents-complete-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-tool-call" - }, - "metric_keys": [], - "name": "mistral-agents-tool-call-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.complete", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "agents-stream" - }, - "metric_keys": [], - "name": "mistral-agents-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-small-2506", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "mistral.agents.stream", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings-create" - }, - "metric_keys": [], - "name": "mistral-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-embed", - "provider": "mistral" - }, - "metric_keys": [ - "completion_tokens", - "prompt_tokens", - "tokens" - ], - "name": "mistral.embeddings.create", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "classifiers-moderate-chat" - }, - "metric_keys": [], - "name": "mistral-classifiers-moderate-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "mistral-moderation-2603", - "provider": "mistral" - }, - "metric_keys": [], - "name": "mistral.classifiers.moderateChat", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.json new file mode 100644 index 000000000..7bde4367b --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.json @@ -0,0 +1,760 @@ +{ + "span_tree": [ + { + "name": "mistral-root", + "type": "task", + "children": [ + { + "name": "mistral-chat-complete-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are concise. Keep responses under five words.", + "role": "system" + }, + { + "content": "Reply with exactly: observability", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "observability", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 24 + } + } + ], + "metadata": { + "operation": "chat-complete", + "testRunId": "" + } + }, + { + "name": "mistral-chat-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: streamed output", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "streamed output", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "mistral-chat-reasoning-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Therefore, older brother's age: \\(11 \\times 2 = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children. Assuming the older brother is not John, the remaining children are the three sisters.\n -", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 256, + "model": "mistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "reasoning_effort": "high", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 256, + "prompt_cached_tokens": 64, + "prompt_tokens": 83, + "time_to_first_token": 0, + "tokens": 339 + } + } + ], + "metadata": { + "operation": "chat-stream-reasoning", + "testRunId": "" + } + }, + { + "name": "mistral-chat-thinking-stream-operation", + "children": [ + { + "name": "mistral.chat.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2+2? Reply with just the number.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "4", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 1024, + "model": "magistral-small-latest", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 28, + "time_to_first_token": 0, + "tokens": 30 + } + } + ], + "metadata": { + "operation": "chat-stream-thinking", + "testRunId": "" + } + }, + { + "name": "mistral-chat-tool-call-operation", + "children": [ + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 13, + "prompt_cached_tokens": 0, + "prompt_tokens": 91, + "time_to_first_token": 0, + "tokens": 104 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 48, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 22, + "prompt_cached_tokens": 64, + "prompt_tokens": 125, + "time_to_first_token": 0, + "tokens": 147 + } + }, + { + "name": "mistral.chat.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "You must return only tool calls and no plain text.", + "role": "system" + }, + { + "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"location\": \"Vienna\"}", + "name": "get_weather" + }, + "id": "", + "index": 0 + }, + { + "function": { + "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + "name": "get_exchange_rate" + }, + "id": "", + "index": 1 + } + ] + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 96, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 34, + "prompt_cached_tokens": 0, + "prompt_tokens": 219, + "time_to_first_token": 0, + "tokens": 253 + } + } + ], + "metadata": { + "operation": "chat-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-fim-complete-operation", + "children": [ + { + "name": "mistral.fim.complete", + "type": "llm", + "children": [], + "input": "function add(a, b) {", + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "\n return a + b;\n", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 24, + "model": "codestral-2508", + "object": "chat.completion", + "provider": "mistral", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 8, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 20 + } + } + ], + "metadata": { + "operation": "fim-complete", + "testRunId": "" + } + }, + { + "name": "mistral-fim-stream-operation", + "children": [ + { + "name": "mistral.fim.stream", + "type": "llm", + "children": [], + "input": "const project = ", + "output": [ + { + "finishReason": "length", + "index": 0, + "message": { + "content": "{\n name: 'project',\n title: 'Projects',\n type:", + "role": "assistant" + } + } + ], + "metadata": { + "created": 0, + "id": "", + "maxTokens": 16, + "model": "codestral-2508", + "object": "chat.completion.chunk", + "provider": "mistral", + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 16, + "prompt_cached_tokens": 0, + "prompt_tokens": 9, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "fim-stream", + "testRunId": "" + } + }, + { + "name": "mistral-agents-complete-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent complete", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Agent complete.", + "prefix": false, + "role": "assistant", + "toolCalls": null + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-complete", + "testRunId": "" + } + }, + { + "name": "mistral-agents-tool-call-operation", + "children": [ + { + "name": "mistral.agents.complete", + "type": "llm", + "children": [], + "input": [ + { + "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "tool_calls", + "index": 0, + "message": { + "content": "", + "prefix": false, + "role": "assistant", + "toolCalls": [ + { + "function": { + "arguments": "{\"city\": \"Vienna\"}", + "name": "get_time_in_city" + }, + "id": "", + "index": 0 + } + ] + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 32, + "model": "mistral-small-2506", + "object": "chat.completion", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "temperature": 0, + "toolChoice": "required" + }, + "metrics": { + "completion_tokens": 14, + "prompt_cached_tokens": 0, + "prompt_tokens": 107, + "time_to_first_token": 0, + "tokens": 121 + } + } + ], + "metadata": { + "operation": "agents-tool-call", + "testRunId": "" + } + }, + { + "name": "mistral-agents-stream-operation", + "children": [ + { + "name": "mistral.agents.stream", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly: agent stream", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "message": { + "content": "Understood.", + "role": "assistant" + } + } + ], + "metadata": { + "agentId": "", + "created": 0, + "id": "", + "maxTokens": 16, + "model": "mistral-small-2506", + "object": "chat.completion.chunk", + "provider": "mistral", + "responseFormat": { + "type": "text" + }, + "stream": true, + "temperature": 0 + }, + "metrics": { + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "agents-stream", + "testRunId": "" + } + }, + { + "name": "mistral-embeddings-operation", + "children": [ + { + "name": "mistral.embeddings.create", + "type": "llm", + "children": [], + "input": "braintrust mistral instrumentation", + "output": { + "embedding_length": 1024 + }, + "metadata": { + "id": "", + "model": "mistral-embed", + "object": "list", + "provider": "mistral" + }, + "metrics": { + "completion_tokens": 0, + "prompt_tokens": 9, + "tokens": 9 + } + } + ], + "metadata": { + "operation": "embeddings-create", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-operation", + "children": [ + { + "name": "mistral.classifiers.moderate", + "type": "llm", + "children": [], + "input": "A short and harmless moderation fixture.", + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.00806199200451374, + "dangerous": 0.005911068990826607, + "financial": 0.00011591934162424877, + "hate_and_discrimination": 0.0023231625091284513, + "health": 0.0003799845289904624, + "jailbreaking": 0.018689308315515518, + "law": 0.00020342697098385543, + "pii": 0.001098731067031622, + "selfharm": 0.0006361911655403674, + "sexual": 0.0031236486975103617, + "violence_and_threats": 0.004609572701156139 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate", + "testRunId": "" + } + }, + { + "name": "mistral-classifiers-moderate-chat-operation", + "children": [ + { + "name": "mistral.classifiers.moderateChat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ], + "output": [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ], + "metadata": { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } + } + ], + "metadata": { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.txt b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.txt new file mode 100644 index 000000000..5c99dd3ea --- /dev/null +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-tree.txt @@ -0,0 +1,640 @@ +span_tree: +└── mistral-root [task] + metadata: { + "scenario": "mistral-instrumentation", + "testRunId": "" + } + ├── mistral-chat-complete-operation + │ metadata: { + │ "operation": "chat-complete", + │ "testRunId": "" + │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You are concise. Keep responses under five words.", + │ "role": "system" + │ }, + │ { + │ "content": "Reply with exactly: observability", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "observability", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 24 + │ } + ├── mistral-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: streamed output", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "streamed output", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 10, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── mistral-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-stream-reasoning", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "Let's break down the problem step by step to find John's age.\n\n1. **First Sister's Age:**\n - Given: The first sister is **4 years old**.\n\n2. **Second Sister's Age Next Year:**\n - Next year, the second sister will be twice as old as the first sister.\n - First sister's age next year: \\(4 + 1 = 5\\) years.\n - Therefore, second sister's age next year: \\(2 \\times 5 = 10\\) years.\n - **Current age of the second sister:** \\(10 - 1 = 9\\) years.\n\n3. **Third Sister's Age:**\n - The third sister is two years older than the second sister.\n - **Current age of the third sister:** \\(9 + 2 = 11\\) years.\n\n4. **Older Brother's Age:**\n - The third sister is half the age of her older brother.\n - Therefore, older brother's age: \\(11 \\times 2 = 22\\) years.\n\n5. **John's Age:**\n - John is one of the 4 children. Assuming the older brother is not John, the remaining children are the three sisters.\n -", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 256, + │ "model": "mistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "reasoning_effort": "high", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 256, + │ "prompt_cached_tokens": 64, + │ "prompt_tokens": 83, + │ "time_to_first_token": 0, + │ "tokens": 339 + │ } + ├── mistral-chat-thinking-stream-operation + │ metadata: { + │ "operation": "chat-stream-thinking", + │ "testRunId": "" + │ } + │ └── mistral.chat.stream [llm] + │ input: [ + │ { + │ "content": "What is 2+2? Reply with just the number.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "4", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 1024, + │ "model": "magistral-small-latest", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 28, + │ "time_to_first_token": 0, + │ "tokens": 30 + │ } + ├── mistral-chat-tool-call-operation + │ metadata: { + │ "operation": "chat-tool-call", + │ "testRunId": "" + │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_weather tool for Vienna. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"location\": \"Vienna\"}", + │ │ "name": "get_weather" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 13, + │ │ "prompt_cached_tokens": 0, + │ │ "prompt_tokens": 91, + │ │ "time_to_first_token": 0, + │ │ "tokens": 104 + │ │ } + │ ├── mistral.chat.complete [llm] + │ │ input: [ + │ │ { + │ │ "content": "Call the get_exchange_rate tool for USD to EUR. Do not answer with plain text.", + │ │ "role": "user" + │ │ } + │ │ ] + │ │ output: [ + │ │ { + │ │ "finishReason": "tool_calls", + │ │ "index": 0, + │ │ "message": { + │ │ "content": "", + │ │ "prefix": false, + │ │ "role": "assistant", + │ │ "toolCalls": [ + │ │ { + │ │ "function": { + │ │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ │ "name": "get_exchange_rate" + │ │ }, + │ │ "id": "", + │ │ "index": 0 + │ │ } + │ │ ] + │ │ } + │ │ } + │ │ ] + │ │ metadata: { + │ │ "created": 0, + │ │ "id": "", + │ │ "maxTokens": 48, + │ │ "model": "mistral-small-2506", + │ │ "object": "chat.completion", + │ │ "provider": "mistral", + │ │ "temperature": 0, + │ │ "toolChoice": "required" + │ │ } + │ │ metrics: { + │ │ "completion_tokens": 22, + │ │ "prompt_cached_tokens": 64, + │ │ "prompt_tokens": 125, + │ │ "time_to_first_token": 0, + │ │ "tokens": 147 + │ │ } + │ └── mistral.chat.complete [llm] + │ input: [ + │ { + │ "content": "You must return only tool calls and no plain text.", + │ "role": "system" + │ }, + │ { + │ "content": "In a single assistant response, call exactly two tools: get_weather with location Vienna and get_exchange_rate with from_currency USD and to_currency EUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"location\": \"Vienna\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "index": 0 + │ }, + │ { + │ "function": { + │ "arguments": "{\"from_currency\": \"USD\", \"to_currency\": \"EUR\"}", + │ "name": "get_exchange_rate" + │ }, + │ "id": "", + │ "index": 1 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 96, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 34, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 219, + │ "time_to_first_token": 0, + │ "tokens": 253 + │ } + ├── mistral-fim-complete-operation + │ metadata: { + │ "operation": "fim-complete", + │ "testRunId": "" + │ } + │ └── mistral.fim.complete [llm] + │ input: "function add(a, b) {" + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "\n return a + b;\n", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 24, + │ "model": "codestral-2508", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 8, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 20 + │ } + ├── mistral-fim-stream-operation + │ metadata: { + │ "operation": "fim-stream", + │ "testRunId": "" + │ } + │ └── mistral.fim.stream [llm] + │ input: "const project = " + │ output: [ + │ { + │ "finishReason": "length", + │ "index": 0, + │ "message": { + │ "content": "{\n name: 'project',\n title: 'Projects',\n type:", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "codestral-2508", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 9, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-complete-operation + │ metadata: { + │ "operation": "agents-complete", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent complete", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Agent complete.", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": null + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-agents-tool-call-operation + │ metadata: { + │ "operation": "agents-tool-call", + │ "testRunId": "" + │ } + │ └── mistral.agents.complete [llm] + │ input: [ + │ { + │ "content": "Call the get_time_in_city tool for Vienna. Do not answer with plain text.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": "", + │ "prefix": false, + │ "role": "assistant", + │ "toolCalls": [ + │ { + │ "function": { + │ "arguments": "{\"city\": \"Vienna\"}", + │ "name": "get_time_in_city" + │ }, + │ "id": "", + │ "index": 0 + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 32, + │ "model": "mistral-small-2506", + │ "object": "chat.completion", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "temperature": 0, + │ "toolChoice": "required" + │ } + │ metrics: { + │ "completion_tokens": 14, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 107, + │ "time_to_first_token": 0, + │ "tokens": 121 + │ } + ├── mistral-agents-stream-operation + │ metadata: { + │ "operation": "agents-stream", + │ "testRunId": "" + │ } + │ └── mistral.agents.stream [llm] + │ input: [ + │ { + │ "content": "Reply with exactly: agent stream", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "message": { + │ "content": "Understood.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "agentId": "", + │ "created": 0, + │ "id": "", + │ "maxTokens": 16, + │ "model": "mistral-small-2506", + │ "object": "chat.completion.chunk", + │ "provider": "mistral", + │ "responseFormat": { + │ "type": "text" + │ }, + │ "stream": true, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── mistral-embeddings-operation + │ metadata: { + │ "operation": "embeddings-create", + │ "testRunId": "" + │ } + │ └── mistral.embeddings.create [llm] + │ input: "braintrust mistral instrumentation" + │ output: { + │ "embedding_length": 1024 + │ } + │ metadata: { + │ "id": "", + │ "model": "mistral-embed", + │ "object": "list", + │ "provider": "mistral" + │ } + │ metrics: { + │ "completion_tokens": 0, + │ "prompt_tokens": 9, + │ "tokens": 9 + │ } + ├── mistral-classifiers-moderate-operation + │ metadata: { + │ "operation": "classifiers-moderate", + │ "testRunId": "" + │ } + │ └── mistral.classifiers.moderate [llm] + │ input: "A short and harmless moderation fixture." + │ output: [ + │ { + │ "categories": { + │ "criminal": false, + │ "dangerous": false, + │ "financial": false, + │ "hate_and_discrimination": false, + │ "health": false, + │ "jailbreaking": false, + │ "law": false, + │ "pii": false, + │ "selfharm": false, + │ "sexual": false, + │ "violence_and_threats": false + │ }, + │ "categoryScores": { + │ "criminal": 0.00806199200451374, + │ "dangerous": 0.005911068990826607, + │ "financial": 0.00011591934162424877, + │ "hate_and_discrimination": 0.0023231625091284513, + │ "health": 0.0003799845289904624, + │ "jailbreaking": 0.018689308315515518, + │ "law": 0.00020342697098385543, + │ "pii": 0.001098731067031622, + │ "selfharm": 0.0006361911655403674, + │ "sexual": 0.0031236486975103617, + │ "violence_and_threats": 0.004609572701156139 + │ } + │ } + │ ] + │ metadata: { + │ "id": "", + │ "model": "mistral-moderation-2603", + │ "provider": "mistral" + │ } + └── mistral-classifiers-moderate-chat-operation + metadata: { + "operation": "classifiers-moderate-chat", + "testRunId": "" + } + └── mistral.classifiers.moderateChat [llm] + input: [ + { + "content": "Please classify this harmless chat message.", + "role": "user" + } + ] + output: [ + { + "categories": { + "criminal": false, + "dangerous": false, + "financial": false, + "hate_and_discrimination": false, + "health": false, + "jailbreaking": false, + "law": false, + "pii": false, + "selfharm": false, + "sexual": false, + "violence_and_threats": false + }, + "categoryScores": { + "criminal": 0.000755405577365309, + "dangerous": 0.00022693384380545467, + "financial": 0.00006402006692951545, + "hate_and_discrimination": 0.0007096704212017357, + "health": 0.00008481103577651083, + "jailbreaking": 0.0030278353951871395, + "law": 0.00011235327838221565, + "pii": 0.000552778597921133, + "selfharm": 0.00004133539550821297, + "sexual": 0.0003799845289904624, + "violence_and_threats": 0.00024923254386521876 + } + } + ] + metadata: { + "id": "", + "model": "mistral-moderation-2603", + "provider": "mistral" + } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index 0fc402522..82f654036 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -1,26 +1,21 @@ import { beforeAll, describe, expect, test } from "vitest"; import type { CapturedLogEvent, - CapturedLogPayload, CapturedLogRow, } from "../../helpers/mock-braintrust-server"; import type { Json } from "../../helpers/normalize"; -import { normalizeForSnapshot } from "../../helpers/normalize"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { effectiveScenarioTimeoutMs, withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; -import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "../../helpers/wrapper-contract"; + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { ADJUSTABLE_REASONING_MODEL, FIM_MODEL, @@ -248,32 +243,6 @@ function normalizeLegacyV134MetricKeys(metricKeys: Json): Json { : metricKeys; } -function normalizeLegacyV134SpanSummaryRow( - summaryRow: Json, - snapshotName: string, -): Json { - if (snapshotName !== "mistral-v1-3-4" || !isRecord(summaryRow)) { - return summaryRow; - } - - const unstableLegacyV134SpanNames = new Set([ - "mistral.chat.stream", - "mistral.fim.stream", - ]); - - if ( - typeof summaryRow.name !== "string" || - !unstableLegacyV134SpanNames.has(summaryRow.name) - ) { - return summaryRow; - } - - return { - ...summaryRow, - metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), - }; -} - function normalizeLegacyV134PayloadSummaryRow( summaryRow: Json, snapshotName: string, @@ -331,67 +300,6 @@ function normalizeLegacyV134PayloadSummaryRow( return normalizedSummaryRow; } -function mergeRecordValues( - left: Record | undefined, - right: unknown, -): Record | undefined { - if (!right || typeof right !== "object" || Array.isArray(right)) { - return left; - } - - return { - ...(left || {}), - ...(right as Record), - }; -} - -function mergePayloadRows(rows: CapturedLogRow[]): CapturedLogRow[] { - const mergedBySpan = new Map(); - const spanOrder: string[] = []; - - for (const row of rows) { - const spanId = - typeof row.span_id === "string" - ? row.span_id - : `unknown-${spanOrder.length}`; - const existing = mergedBySpan.get(spanId); - - if (!existing) { - mergedBySpan.set(spanId, { - ...row, - }); - spanOrder.push(spanId); - continue; - } - - mergedBySpan.set(spanId, { - ...existing, - ...(row.input !== undefined && row.input !== null - ? { - input: row.input, - } - : {}), - ...(row.output !== undefined && row.output !== null - ? { - output: row.output, - } - : {}), - metadata: mergeRecordValues( - existing.metadata as Record | undefined, - row.metadata, - ), - metrics: mergeRecordValues( - existing.metrics as Record | undefined, - row.metrics, - ), - }); - } - - return spanOrder - .map((spanId) => mergedBySpan.get(spanId)) - .filter((row): row is CapturedLogRow => row !== undefined); -} - function pickOutputSpans(spans: T[]): T[] { const spansWithOutput = spans.filter( (span) => span.output !== undefined && span.output !== null, @@ -399,10 +307,7 @@ function pickOutputSpans(spans: T[]): T[] { return spansWithOutput.length > 0 ? spansWithOutput : spans; } -function buildSpanSummary( - events: CapturedLogEvent[], - snapshotName: string, -): Json { +function snapshotEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { const chatCompleteOperation = findLatestSpan( events, "mistral-chat-complete-operation", @@ -476,170 +381,96 @@ function buildSpanSummary( "mistral-classifiers-classify-chat-operation", ); - return normalizeForSnapshot( - [ - findLatestSpan(events, ROOT_NAME), - chatCompleteOperation, - findMistralSpan(events, chatCompleteOperation?.span.id, [ - "mistral.chat.complete", - ]), - chatStreamOperation, - findMistralSpan(events, chatStreamOperation?.span.id, [ - "mistral.chat.stream", - ]), - chatReasoningStreamOperation, - findMistralSpan(events, chatReasoningStreamOperation?.span.id, [ - "mistral.chat.stream", - ]), - chatThinkingStreamOperation, - findMistralSpan(events, chatThinkingStreamOperation?.span.id, [ - "mistral.chat.stream", - ]), - chatToolCallOperation, - ...selectedChatToolCallSpans, - fimCompleteOperation, - findMistralSpan(events, fimCompleteOperation?.span.id, [ - "mistral.fim.complete", - ]), - fimStreamOperation, - findMistralSpan(events, fimStreamOperation?.span.id, [ - "mistral.fim.stream", - ]), - agentsCompleteOperation, - findMistralSpan(events, agentsCompleteOperation?.span.id, [ - "mistral.agents.complete", - ]), - agentsToolCallOperation, - ...selectedAgentsToolCallSpans, - agentsStreamOperation, - findMistralSpan(events, agentsStreamOperation?.span.id, [ - "mistral.agents.stream", - ]), - embeddingsOperation, - findMistralSpan(events, embeddingsOperation?.span.id, [ - "mistral.embeddings.create", - ]), - classifiersModerateOperation, - findMistralSpan(events, classifiersModerateOperation?.span.id, [ - "mistral.classifiers.moderate", - ]), - classifiersModerateChatOperation, - findMistralSpan(events, classifiersModerateChatOperation?.span.id, [ - "mistral.classifiers.moderateChat", - ]), - classifiersClassifyOperation, - findMistralSpan(events, classifiersClassifyOperation?.span.id, [ - "mistral.classifiers.classify", - ]), - classifiersClassifyChatOperation, - findMistralSpan(events, classifiersClassifyChatOperation?.span.id, [ - "mistral.classifiers.classifyChat", - ]), - ] - .filter((event): event is CapturedLogEvent => event !== undefined) - .map((event) => - normalizeLegacyV134SpanSummaryRow( - summarizeWrapperContract(event, [ - "model", - "operation", - "provider", - "reasoning_effort", - "scenario", - ]), - snapshotName, - ), - ) as Json, - ); + return [ + findLatestSpan(events, ROOT_NAME), + chatCompleteOperation, + findMistralSpan(events, chatCompleteOperation?.span.id, [ + "mistral.chat.complete", + ]), + chatStreamOperation, + findMistralSpan(events, chatStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + chatReasoningStreamOperation, + findMistralSpan(events, chatReasoningStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + chatThinkingStreamOperation, + findMistralSpan(events, chatThinkingStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + chatToolCallOperation, + ...selectedChatToolCallSpans, + fimCompleteOperation, + findMistralSpan(events, fimCompleteOperation?.span.id, [ + "mistral.fim.complete", + ]), + fimStreamOperation, + findMistralSpan(events, fimStreamOperation?.span.id, [ + "mistral.fim.stream", + ]), + agentsCompleteOperation, + findMistralSpan(events, agentsCompleteOperation?.span.id, [ + "mistral.agents.complete", + ]), + agentsToolCallOperation, + ...selectedAgentsToolCallSpans, + agentsStreamOperation, + findMistralSpan(events, agentsStreamOperation?.span.id, [ + "mistral.agents.stream", + ]), + embeddingsOperation, + findMistralSpan(events, embeddingsOperation?.span.id, [ + "mistral.embeddings.create", + ]), + classifiersModerateOperation, + findMistralSpan(events, classifiersModerateOperation?.span.id, [ + "mistral.classifiers.moderate", + ]), + classifiersModerateChatOperation, + findMistralSpan(events, classifiersModerateChatOperation?.span.id, [ + "mistral.classifiers.moderateChat", + ]), + classifiersClassifyOperation, + findMistralSpan(events, classifiersClassifyOperation?.span.id, [ + "mistral.classifiers.classify", + ]), + classifiersClassifyChatOperation, + findMistralSpan(events, classifiersClassifyChatOperation?.span.id, [ + "mistral.classifiers.classifyChat", + ]), + ].filter((event): event is CapturedLogEvent => event !== undefined); } -function buildPayloadSummary( +function buildSpanTree( events: CapturedLogEvent[], - payloads: CapturedLogPayload[], snapshotName: string, -): Json { - const parentIdBySpanId = new Map(); - const spanNameById = new Map(); - for (const event of events) { - if ( - typeof event?.span?.id === "string" && - typeof event?.span?.name === "string" - ) { - const parentId = - Array.isArray(event.span.parentIds) && - typeof event.span.parentIds[0] === "string" - ? event.span.parentIds[0] - : ""; - parentIdBySpanId.set(event.span.id, parentId); - spanNameById.set(event.span.id, event.span.name); - } - } +): SpanTreeEntry[] { + return snapshotEvents(events).map((event) => { + const summary = normalizeLegacyV134PayloadSummaryRow( + summarizePayloadRow(event.row), + snapshotName, + event.span.name, + ) as Record; + const { + span_id: _spanId, + has_input: _hasInput, + has_output: _hasOutput, + metric_keys: metricKeys, + ...fields + } = summary; - const root = findLatestSpan(events, ROOT_NAME); - const payloadRows = payloadRowsForRootSpan(payloads, root?.span.id); - const mergedRows = mergePayloadRows(payloadRows); - const llmSpanNames = new Set([ - "mistral.chat.complete", - "mistral.chat.stream", - "mistral.fim.complete", - "mistral.fim.stream", - "mistral.agents.complete", - "mistral.agents.stream", - "mistral.embeddings.create", - "mistral.classifiers.moderate", - "mistral.classifiers.moderateChat", - "mistral.classifiers.classify", - "mistral.classifiers.classifyChat", - ]); - const parentAndNameWithOutput = new Set(); - - for (const row of mergedRows) { - if (typeof row.span_id !== "string") { - continue; - } - const spanName = spanNameById.get(row.span_id); - if (!spanName || !llmSpanNames.has(spanName)) { - continue; - } - if (row.output === undefined || row.output === null) { - continue; - } - - const parentId = parentIdBySpanId.get(row.span_id) || ""; - parentAndNameWithOutput.add(`${parentId}:${spanName}`); - } - - const stableRows = mergedRows.filter((row) => { - if (typeof row.span_id !== "string") { - return true; - } - - const spanName = spanNameById.get(row.span_id); - if (!spanName || !llmSpanNames.has(spanName)) { - return true; - } - - if (row.output !== undefined && row.output !== null) { - return true; - } - - const parentId = parentIdBySpanId.get(row.span_id) || ""; - return !parentAndNameWithOutput.has(`${parentId}:${spanName}`); + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + metrics: { + keys: metricKeys, + }, + }, + }; }); - - return normalizeForSnapshot( - stableRows.map((row) => { - const spanName = - typeof row.span_id === "string" - ? spanNameById.get(row.span_id) - : undefined; - - return normalizeLegacyV134PayloadSummaryRow( - summarizePayloadRow(row), - snapshotName, - spanName, - ); - }), - ); } export function defineMistralInstrumentationAssertions(options: { @@ -654,11 +485,7 @@ export function defineMistralInstrumentationAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const supportsThinkingStream = options.supportsThinkingStream ?? true; const supportsClassifiers = options.supportsClassifiers ?? true; @@ -672,13 +499,11 @@ export function defineMistralInstrumentationAssertions(options: { describe(options.name, () => { let events: CapturedLogEvent[] = []; - let payloads: CapturedLogPayload[] = []; beforeAll(async () => { await withScenarioHarness(async (harness) => { await options.runScenario(harness); events = harness.events(); - payloads = harness.payloads(); }); }, timeoutMs); @@ -1208,20 +1033,8 @@ export function defineMistralInstrumentationAssertions(options: { }); } - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events, options.snapshotName)), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildPayloadSummary(events, payloads, options.snapshotName), - ), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-events.json b/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-events.json deleted file mode 100644 index 28f671eb2..000000000 --- a/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-events.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "error": null, - "input": { - "route": "/api/smoke-test/edge", - "runtime": "edge" - }, - "metadata": { - "runtime": "edge", - "scenario": "nextjs-instrumentation", - "testRunId": "", - "transport": "http" - }, - "name": "nextjs edge logger span", - "output": { - "ok": true, - "route": "/api/smoke-test/edge", - "runtime": "edge" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "nextjs edge logger span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "route": "/api/smoke-test/node", - "runtime": "nodejs" - }, - "metadata": { - "runtime": "nodejs", - "scenario": "nextjs-instrumentation", - "testRunId": "", - "transport": "http" - }, - "name": "nextjs nodejs logger span", - "output": { - "ok": true, - "route": "/api/smoke-test/node", - "runtime": "nodejs" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "nextjs nodejs logger span", - "type": "task" - }, - "span_id": "", - "span_parents": null - } -] diff --git a/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.json b/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.json new file mode 100644 index 000000000..b04557736 --- /dev/null +++ b/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.json @@ -0,0 +1,44 @@ +{ + "span_tree": [ + { + "name": "nextjs nodejs logger span", + "type": "task", + "children": [], + "input": { + "route": "/api/smoke-test/node", + "runtime": "nodejs" + }, + "output": { + "ok": true, + "route": "/api/smoke-test/node", + "runtime": "nodejs" + }, + "metadata": { + "runtime": "nodejs", + "scenario": "nextjs-instrumentation", + "testRunId": "", + "transport": "http" + } + }, + { + "name": "nextjs edge logger span", + "type": "task", + "children": [], + "input": { + "route": "/api/smoke-test/edge", + "runtime": "edge" + }, + "output": { + "ok": true, + "route": "/api/smoke-test/edge", + "runtime": "edge" + }, + "metadata": { + "runtime": "edge", + "scenario": "nextjs-instrumentation", + "testRunId": "", + "transport": "http" + } + } + ] +} diff --git a/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.txt b/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.txt new file mode 100644 index 000000000..2b7a65555 --- /dev/null +++ b/e2e/scenarios/nextjs-instrumentation/__snapshots__/span-tree.txt @@ -0,0 +1,33 @@ +span_tree: +├── nextjs nodejs logger span [task] +│ input: { +│ "route": "/api/smoke-test/node", +│ "runtime": "nodejs" +│ } +│ output: { +│ "ok": true, +│ "route": "/api/smoke-test/node", +│ "runtime": "nodejs" +│ } +│ metadata: { +│ "runtime": "nodejs", +│ "scenario": "nextjs-instrumentation", +│ "testRunId": "", +│ "transport": "http" +│ } +└── nextjs edge logger span [task] + input: { + "route": "/api/smoke-test/edge", + "runtime": "edge" + } + output: { + "ok": true, + "route": "/api/smoke-test/edge", + "runtime": "edge" + } + metadata: { + "runtime": "edge", + "scenario": "nextjs-instrumentation", + "testRunId": "", + "transport": "http" + } diff --git a/e2e/scenarios/nextjs-instrumentation/scenario.test.ts b/e2e/scenarios/nextjs-instrumentation/scenario.test.ts index c654794f0..7e16076af 100644 --- a/e2e/scenarios/nextjs-instrumentation/scenario.test.ts +++ b/e2e/scenarios/nextjs-instrumentation/scenario.test.ts @@ -10,9 +10,9 @@ import { resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { extractOtelSpans, - summarizeEvent, summarizeRequest, } from "../../helpers/trace-summary"; import { findLatestSpan } from "../../helpers/trace-selectors"; @@ -202,11 +202,9 @@ test( resolveFileSnapshotPath(import.meta.url, "route-responses.json"), ); - await matchFileSnapshot( - formatJsonFileSnapshot( - [edgeSpan, nodeSpan].map((event) => summarizeEvent(event!)) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "span-events.json"), + await matchSpanTreeSnapshot( + events, + resolveFileSnapshotPath(import.meta.url, "span-tree.json"), ); await matchFileSnapshot( diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-events.json b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-events.json deleted file mode 100644 index 7423b10f3..000000000 --- a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-events.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "llms": [ - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.llm_sequence": 1, - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.run", - "provider": "openai" - }, - "metric_keys": [], - "name": "OpenAI Codex LLM", - "output": { - "message": "OPENAI_CODEX_RUN_OK" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.llm_sequence": 1, - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.runStreamed", - "openai_codex.thread_id": "", - "provider": "openai" - }, - "metric_keys": [], - "name": "OpenAI Codex LLM", - "output": { - "message": "OPENAI_CODEX_STREAM_OK" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } - ], - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "openai-codex-instrumentation" - }, - "metric_keys": [], - "name": "openai-codex-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "run": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "run" - }, - "metric_keys": [], - "name": "openai-codex-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.run", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "duration", - "prompt_cached_tokens", - "prompt_tokens" - ], - "name": "OpenAI Codex", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "streamed": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "runStreamed" - }, - "metric_keys": [], - "name": "openai-codex-run-streamed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.runStreamed", - "openai_codex.thread_id": "", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "duration", - "prompt_cached_tokens", - "prompt_tokens" - ], - "name": "OpenAI Codex", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "tools": [ - "tool: web_search" - ] -} diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.json b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.json new file mode 100644 index 000000000..49a8c6d27 --- /dev/null +++ b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.json @@ -0,0 +1,148 @@ +{ + "span_tree": [ + { + "name": "openai-codex-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-codex-run-operation", + "children": [ + { + "name": "OpenAI Codex", + "type": "task", + "children": [ + { + "name": "tool: web_search", + "type": "tool", + "children": [], + "input": "Braintrust SDK JavaScript", + "metadata": { + "gen_ai.tool.name": "web_search", + "openai_codex.item_id": "ws_0325f84104f7f2de016a073a98a5288190a8e9b0427e2d92d7", + "openai_codex.item_type": "web_search" + } + }, + { + "name": "OpenAI Codex LLM", + "type": "llm", + "children": [], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK", + "output": { + "message": "OPENAI_CODEX_RUN_OK" + }, + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.run", + "provider": "openai" + } + } + ], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK", + "output": "OPENAI_CODEX_RUN_OK", + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.run", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.version": "0.128.0", + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/run", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 64, + "completion_tokens": 135, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 19578 + } + } + ], + "metadata": { + "operation": "run", + "testRunId": "" + } + }, + { + "name": "openai-codex-run-streamed-operation", + "children": [ + { + "name": "OpenAI Codex", + "type": "task", + "children": [ + { + "name": "tool: web_search", + "type": "tool", + "children": [], + "input": "", + "metadata": { + "gen_ai.tool.name": "web_search", + "openai_codex.item_id": "ws_034e3e5eb5b136a7016a073aa00e648194b34f84887a101c22", + "openai_codex.item_type": "web_search" + } + }, + { + "name": "OpenAI Codex LLM", + "type": "llm", + "children": [], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK", + "output": { + "message": "OPENAI_CODEX_STREAM_OK" + }, + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.thread_id": "", + "provider": "openai" + } + } + ], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK", + "output": "OPENAI_CODEX_STREAM_OK", + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.thread_id": "", + "openai_codex.version": "0.128.0", + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/stream", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 118, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17578 + } + } + ], + "metadata": { + "operation": "runStreamed", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "openai-codex-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.txt b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.txt new file mode 100644 index 000000000..ddee88777 --- /dev/null +++ b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-auto-hook.span-tree.txt @@ -0,0 +1,106 @@ +span_tree: +└── openai-codex-instrumentation-root [task] + metadata: { + "scenario": "openai-codex-instrumentation", + "testRunId": "" + } + ├── openai-codex-run-operation + │ metadata: { + │ "operation": "run", + │ "testRunId": "" + │ } + │ └── OpenAI Codex [task] + │ input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK" + │ output: "OPENAI_CODEX_RUN_OK" + │ metadata: { + │ "model": "gpt-5.1-codex-mini", + │ "openai_codex.approval_policy": "never", + │ "openai_codex.model": "gpt-5.1-codex-mini", + │ "openai_codex.model_reasoning_effort": "low", + │ "openai_codex.network_access_enabled": false, + │ "openai_codex.operation": "Thread.run", + │ "openai_codex.sandbox_mode": "read-only", + │ "openai_codex.skip_git_repo_check": true, + │ "openai_codex.version": "0.128.0", + │ "openai_codex.web_search_mode": "live", + │ "openai_codex.working_directory": "/braintrust-codex-e2e/run", + │ "provider": "openai" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 64, + │ "completion_tokens": 135, + │ "duration": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 19578 + │ } + │ ├── tool: web_search [tool] + │ │ input: "Braintrust SDK JavaScript" + │ │ metadata: { + │ │ "gen_ai.tool.name": "web_search", + │ │ "openai_codex.item_id": "ws_0325f84104f7f2de016a073a98a5288190a8e9b0427e2d92d7", + │ │ "openai_codex.item_type": "web_search" + │ │ } + │ └── OpenAI Codex LLM [llm] + │ input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK" + │ output: { + │ "message": "OPENAI_CODEX_RUN_OK" + │ } + │ metadata: { + │ "model": "gpt-5.1-codex-mini", + │ "openai_codex.llm_sequence": 1, + │ "openai_codex.model": "gpt-5.1-codex-mini", + │ "openai_codex.model_reasoning_effort": "low", + │ "openai_codex.operation": "Thread.run", + │ "provider": "openai" + │ } + └── openai-codex-run-streamed-operation + metadata: { + "operation": "runStreamed", + "testRunId": "" + } + └── OpenAI Codex [task] + input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK" + output: "OPENAI_CODEX_STREAM_OK" + metadata: { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.thread_id": "", + "openai_codex.version": "0.128.0", + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/stream", + "provider": "openai" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 118, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17578 + } + ├── tool: web_search [tool] + │ input: "" + │ metadata: { + │ "gen_ai.tool.name": "web_search", + │ "openai_codex.item_id": "ws_034e3e5eb5b136a7016a073aa00e648194b34f84887a101c22", + │ "openai_codex.item_type": "web_search" + │ } + └── OpenAI Codex LLM [llm] + input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK" + output: { + "message": "OPENAI_CODEX_STREAM_OK" + } + metadata: { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.thread_id": "", + "provider": "openai" + } diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-events.json b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-events.json deleted file mode 100644 index 7423b10f3..000000000 --- a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-events.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "llms": [ - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.llm_sequence": 1, - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.run", - "provider": "openai" - }, - "metric_keys": [], - "name": "OpenAI Codex LLM", - "output": { - "message": "OPENAI_CODEX_RUN_OK" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.llm_sequence": 1, - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.runStreamed", - "openai_codex.thread_id": "", - "provider": "openai" - }, - "metric_keys": [], - "name": "OpenAI Codex LLM", - "output": { - "message": "OPENAI_CODEX_STREAM_OK" - }, - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } - ], - "root": { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "openai-codex-instrumentation" - }, - "metric_keys": [], - "name": "openai-codex-instrumentation-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - "run": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "run" - }, - "metric_keys": [], - "name": "openai-codex-run-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.run", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "duration", - "prompt_cached_tokens", - "prompt_tokens" - ], - "name": "OpenAI Codex", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "streamed": { - "operation": { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "runStreamed" - }, - "metric_keys": [], - "name": "openai-codex-run-streamed-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - "task": { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-5.1-codex-mini", - "openai_codex.model": "gpt-5.1-codex-mini", - "openai_codex.operation": "Thread.runStreamed", - "openai_codex.thread_id": "", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "duration", - "prompt_cached_tokens", - "prompt_tokens" - ], - "name": "OpenAI Codex", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } - }, - "tools": [ - "tool: web_search" - ] -} diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.json b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.json new file mode 100644 index 000000000..6c9d69150 --- /dev/null +++ b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.json @@ -0,0 +1,146 @@ +{ + "span_tree": [ + { + "name": "openai-codex-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-codex-run-operation", + "children": [ + { + "name": "OpenAI Codex", + "type": "task", + "children": [ + { + "name": "tool: web_search", + "type": "tool", + "children": [], + "input": "Braintrust SDK JavaScript", + "metadata": { + "gen_ai.tool.name": "web_search", + "openai_codex.item_id": "ws_05904bb998ba1d8d016a073a8c208c8195835f574484fb1970", + "openai_codex.item_type": "web_search" + } + }, + { + "name": "OpenAI Codex LLM", + "type": "llm", + "children": [], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK", + "output": { + "message": "OPENAI_CODEX_RUN_OK" + }, + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.run", + "provider": "openai" + } + } + ], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK", + "output": "OPENAI_CODEX_RUN_OK", + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.run", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/run", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 116, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 19669 + } + } + ], + "metadata": { + "operation": "run", + "testRunId": "" + } + }, + { + "name": "openai-codex-run-streamed-operation", + "children": [ + { + "name": "OpenAI Codex", + "type": "task", + "children": [ + { + "name": "tool: web_search", + "type": "tool", + "children": [], + "input": "", + "metadata": { + "gen_ai.tool.name": "web_search", + "openai_codex.item_id": "ws_0308fe2dfb626b91016a073a90d8dc8197bc4c0cc4f1749444", + "openai_codex.item_type": "web_search" + } + }, + { + "name": "OpenAI Codex LLM", + "type": "llm", + "children": [], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK", + "output": { + "message": "OPENAI_CODEX_STREAM_OK" + }, + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.thread_id": "", + "provider": "openai" + } + } + ], + "input": "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK", + "output": "OPENAI_CODEX_STREAM_OK", + "metadata": { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.thread_id": "", + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/stream", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 77, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 19654 + } + } + ], + "metadata": { + "operation": "runStreamed", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "openai-codex-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.txt b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.txt new file mode 100644 index 000000000..b4911d0ea --- /dev/null +++ b/e2e/scenarios/openai-codex-instrumentation/__snapshots__/openai-codex-v0128-wrapped.span-tree.txt @@ -0,0 +1,104 @@ +span_tree: +└── openai-codex-instrumentation-root [task] + metadata: { + "scenario": "openai-codex-instrumentation", + "testRunId": "" + } + ├── openai-codex-run-operation + │ metadata: { + │ "operation": "run", + │ "testRunId": "" + │ } + │ └── OpenAI Codex [task] + │ input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK" + │ output: "OPENAI_CODEX_RUN_OK" + │ metadata: { + │ "model": "gpt-5.1-codex-mini", + │ "openai_codex.approval_policy": "never", + │ "openai_codex.model": "gpt-5.1-codex-mini", + │ "openai_codex.model_reasoning_effort": "low", + │ "openai_codex.network_access_enabled": false, + │ "openai_codex.operation": "Thread.run", + │ "openai_codex.sandbox_mode": "read-only", + │ "openai_codex.skip_git_repo_check": true, + │ "openai_codex.web_search_mode": "live", + │ "openai_codex.working_directory": "/braintrust-codex-e2e/run", + │ "provider": "openai" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 116, + │ "duration": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 19669 + │ } + │ ├── tool: web_search [tool] + │ │ input: "Braintrust SDK JavaScript" + │ │ metadata: { + │ │ "gen_ai.tool.name": "web_search", + │ │ "openai_codex.item_id": "ws_05904bb998ba1d8d016a073a8c208c8195835f574484fb1970", + │ │ "openai_codex.item_type": "web_search" + │ │ } + │ └── OpenAI Codex LLM [llm] + │ input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_RUN_OK" + │ output: { + │ "message": "OPENAI_CODEX_RUN_OK" + │ } + │ metadata: { + │ "model": "gpt-5.1-codex-mini", + │ "openai_codex.llm_sequence": 1, + │ "openai_codex.model": "gpt-5.1-codex-mini", + │ "openai_codex.model_reasoning_effort": "low", + │ "openai_codex.operation": "Thread.run", + │ "provider": "openai" + │ } + └── openai-codex-run-streamed-operation + metadata: { + "operation": "runStreamed", + "testRunId": "" + } + └── OpenAI Codex [task] + input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK" + output: "OPENAI_CODEX_STREAM_OK" + metadata: { + "model": "gpt-5.1-codex-mini", + "openai_codex.approval_policy": "never", + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.network_access_enabled": false, + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.sandbox_mode": "read-only", + "openai_codex.skip_git_repo_check": true, + "openai_codex.thread_id": "", + "openai_codex.web_search_mode": "live", + "openai_codex.working_directory": "/braintrust-codex-e2e/stream", + "provider": "openai" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 77, + "duration": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 19654 + } + ├── tool: web_search [tool] + │ input: "" + │ metadata: { + │ "gen_ai.tool.name": "web_search", + │ "openai_codex.item_id": "ws_0308fe2dfb626b91016a073a90d8dc8197bc4c0cc4f1749444", + │ "openai_codex.item_type": "web_search" + │ } + └── OpenAI Codex LLM [llm] + input: "You are running inside an SDK instrumentation test. Use web search once for the exact query \"Braintrust SDK JavaScript\". Do not run shell commands or inspect local files. After the web search completes, reply with exactly this marker and no extra text: OPENAI_CODEX_STREAM_OK" + output: { + "message": "OPENAI_CODEX_STREAM_OK" + } + metadata: { + "model": "gpt-5.1-codex-mini", + "openai_codex.llm_sequence": 1, + "openai_codex.model": "gpt-5.1-codex-mini", + "openai_codex.model_reasoning_effort": "low", + "openai_codex.operation": "Thread.runStreamed", + "openai_codex.thread_id": "", + "provider": "openai" + } diff --git a/e2e/scenarios/openai-codex-instrumentation/assertions.ts b/e2e/scenarios/openai-codex-instrumentation/assertions.ts index e28adf9a1..5d3060f64 100644 --- a/e2e/scenarios/openai-codex-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-codex-instrumentation/assertions.ts @@ -1,17 +1,17 @@ import { beforeAll, describe, expect, test } from "vitest"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, + type SpanTreeFields, +} from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; type RunOpenAICodexScenario = (harness: { @@ -160,24 +160,33 @@ function sequenceNumber(event: CapturedLogEvent): number | undefined { return typeof value === "number" ? value : undefined; } -function summarizeSpan(event: CapturedLogEvent | undefined): Json { - if (!event) { - return null; - } - const summary = summarizeWrapperContract(event, [...METADATA_KEYS]) as Record< - string, - Json - >; - if (summary.metadata && typeof summary.metadata === "object") { - const metadata = summary.metadata as Record; - if (typeof metadata["openai_codex.thread_id"] === "string") { - metadata["openai_codex.thread_id"] = ""; - } +function snapshotFields(event: CapturedLogEvent): SpanTreeFields { + const fields = spanTreeFields(event); + const metadata = + fields.metadata && + typeof fields.metadata === "object" && + !Array.isArray(fields.metadata) + ? Object.fromEntries( + Object.entries(fields.metadata).filter(([key]) => + METADATA_KEYS.includes(key as (typeof METADATA_KEYS)[number]), + ), + ) + : undefined; + + if (metadata && typeof metadata["openai_codex.thread_id"] === "string") { + metadata["openai_codex.thread_id"] = ""; } - return summary; + + return { + ...fields, + metadata, + ...(event.span.type === "llm" + ? { output: summarizeLlmOutput(event.output) } + : {}), + }; } -function summarizeLlmOutput(output: unknown): Json { +function summarizeLlmOutput(output: unknown): SpanTreeFields["output"] { if (typeof output !== "object" || output === null || Array.isArray(output)) { return null; } @@ -190,22 +199,10 @@ function summarizeLlmOutput(output: unknown): Json { ...(typeof outputRecord.message === "string" ? { message: outputRecord.message } : {}), - } as Json; -} - -function summarizeLlmSpan(event: CapturedLogEvent | undefined): Json { - const summary = summarizeSpan(event) as Record; - summary.output = summarizeLlmOutput(event?.output); - return summary as Json; -} - -function summarizeToolNames(events: CapturedLogEvent[]): Json { - return [...new Set(events.map((event) => event.span.name))] - .filter((name): name is string => typeof name === "string") - .sort(); + }; } -function summarize(events: CapturedLogEvent[]): Json { +function summarize(events: CapturedLogEvent[]): SpanTreeEntry[] { const runTask = findCodexTask(events, "openai-codex-run-operation"); const streamedTask = findCodexTask( events, @@ -214,23 +211,17 @@ function summarize(events: CapturedLogEvent[]): Json { const llmSpans = latestSpansByType(events, "llm"); const toolSpans = latestSpansByType(events, "tool"); - return normalizeForSnapshot({ - root: summarizeSpan(findLatestSpan(events, ROOT_NAME)), - run: { - operation: summarizeSpan( - findLatestSpan(events, "openai-codex-run-operation"), - ), - task: summarizeSpan(runTask), - }, - streamed: { - operation: summarizeSpan( - findLatestSpan(events, "openai-codex-run-streamed-operation"), - ), - task: summarizeSpan(streamedTask), - }, - llms: llmSpans.map(summarizeLlmSpan), - tools: summarizeToolNames(toolSpans), - } as Json); + return [ + findLatestSpan(events, ROOT_NAME), + findLatestSpan(events, "openai-codex-run-operation"), + runTask, + findLatestSpan(events, "openai-codex-run-streamed-operation"), + streamedTask, + ...llmSpans, + ...toolSpans, + ].flatMap((event) => + event ? [{ event, fields: snapshotFields(event) }] : [], + ); } function spanSnapshotPath(options: { @@ -239,7 +230,7 @@ function spanSnapshotPath(options: { }): string { return resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); } @@ -342,11 +333,8 @@ export function defineOpenAICodexInstrumentationAssertions(options: { } }); - test("matches the span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(summarize(events)), - spanSnapshotPath(options), - ); + test("matches the span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath(options)); }); }); } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json new file mode 100644 index 000000000..c22972216 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -0,0 +1,1219 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt new file mode 100644 index 000000000..491388e2d --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -0,0 +1,1081 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + └── openai-responses-parse-operation + metadata: { + "operation": "responses-parse", + "testRunId": "" + } + └── openai.responses.create [llm] + input: "What is 20 + 4?" + output: [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ] + metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json new file mode 100644 index 000000000..1fd5c8904 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -0,0 +1,1069 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.parse", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt new file mode 100644 index 000000000..7352b8b24 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -0,0 +1,941 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + └── openai-responses-parse-operation + metadata: { + "operation": "responses-parse", + "testRunId": "" + } + └── openai.responses.parse [llm] + input: "What is 20 + 4?" + output: [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ] + metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives a total of 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 22, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 68 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.log-payloads.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.log-payloads.json deleted file mode 100644 index 787aa05f5..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.log-payloads.json +++ /dev/null @@ -1,578 +0,0 @@ -[ - { - "input": { - "kind": "undefined" - }, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "output": null, - "type": "task" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [ - "answer" - ], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "output": { - "embedding_length": 1536 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "output": { - "flagged_count": 0, - "result_count": 1 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.span-events.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.span-events.json deleted file mode 100644 index 751f75477..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v4.span-events.json +++ /dev/null @@ -1,390 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json new file mode 100644 index 000000000..4f84bf8f9 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -0,0 +1,1219 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 20, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 66 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt new file mode 100644 index 000000000..9eda1e206 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -0,0 +1,1081 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + └── openai-responses-parse-operation + metadata: { + "operation": "responses-parse", + "testRunId": "" + } + └── openai.responses.create [llm] + input: "What is 20 + 4?" + output: [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ] + metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 20, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 66 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json new file mode 100644 index 000000000..5c976b92a --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -0,0 +1,1069 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS.", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "RESPONSE STREAM", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.parse", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 20, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 66 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt new file mode 100644 index 000000000..1c1129de6 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -0,0 +1,941 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS.", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "RESPONSE STREAM", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + └── openai-responses-parse-operation + metadata: { + "operation": "responses-parse", + "testRunId": "" + } + └── openai.responses.parse [llm] + input: "What is 20 + 4?" + output: [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 together gives 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ] + metadata: { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 together gives 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 together gives 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 20, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 66 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.log-payloads.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.log-payloads.json deleted file mode 100644 index 787aa05f5..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.log-payloads.json +++ /dev/null @@ -1,578 +0,0 @@ -[ - { - "input": { - "kind": "undefined" - }, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "output": null, - "type": "task" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [ - "answer" - ], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "output": { - "embedding_length": 1536 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "output": { - "flagged_count": 0, - "result_count": 1 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.span-events.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.span-events.json deleted file mode 100644 index 751f75477..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v5.span-events.json +++ /dev/null @@ -1,390 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json new file mode 100644 index 000000000..ef239b096 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -0,0 +1,1288 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "Sure! What would you like to know or discuss?", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 12, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 23, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 69 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + }, + { + "name": "openai-responses-compact-operation", + "children": [ + { + "name": "openai.responses.compact", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "id": "", + "role": "user", + "status": "completed", + "type": "message" + }, + { + "encrypted_content": "gAAAAABqBwV0tsDVAxoc2eIrNCzqFQMDcx_WtLB7jn1xjJshXoJiwnwd5Cu4dhaqCh9Xcc-MmsfZwlulYwKvJyc_9GMFb4a3hoZc-2P_IVhiMB3WBin89wlcd1L45sLcJ96FLirt0QD85Qq06DfX4ulYI_NiTA4jBtV2gZ-4EfubN4C8DrHHOvaQeq9twx6qNCEXCDOnJCeVC1QiblKYlQQmW4LEs8ej6QDonbxgwRZty2GxIXWXIzorpB9BD-JBlz9NruYz1gVJibftE3ItME06-Ej6ZTaGXvIYejuba9GWoIRVp4M74mBuWrD1cuvHHBOzLtOK0DD4VfuO7rYJkhrydbtBBEiG6KgmxuwAFNNcjUh8hlpdMi407ROMUhU4GhRq_t8lOJwTQmetb-U35B_DJqIiF5H2ub4_MNTfp732hqEUg51dAl4o4Ms75IMkApTgqwl1C_CnE_aIDkYtsGlVZo97M_2zS5DZ_aTzm2KrqKOM_fS68WZE4XZR8f5LaZyMnN5avHvl_dxE-tTsl_8SeyaLn3rj8mfEetv-Kf_Odld0fsNebcgakB8cZQ4tj-55guNDw3qqNoSXBCbnytk3Mr_WgdhzMofhZ3UrA6Vm1c5eyTtlX4GMwV09wgj-AcP48vfoOGJj-qNurjsGtdn30FzRlBUdGXExReHS0A2nr2-X3zcCkcjbEMoXjcHljpLbeuzkNRb6e5UyAQvQ0aIqa084U92k4icTyhlcHYPA2cVAap_P5c3qkkyZsiYBg13lzdKAkwMzpo6oznapLrhfwrvGIIW6k3dYEqMy5jfIZnrYz6BWWFuV2E9Iwg6R1tX8v3u2lG062YHgWSdT7uHk-ut68J-ilX92jbeWpy_1ML-sc866lstqW5qR_ogotAUWlIk8Xsw_ahN2HcGjVJuXzgwVU9QoaW8pwCimBfGF-dcP2-GvxTxBgGZSJe7iwA5wwFGEVgNE9Xrd3ESsTTTJyG_WLoAiBbZwWkIn68BYQzxJIWCJgXIF9-8BrGR5bD075IhFcMi9XTBdj3uzVjaDxv1t7_opxMXkxisZmCNxFcYuQZgOdWeRh07GtknWnDKJnPriVh17Rg1o8Yz8zGBHhoF2qgpmgwah7Gh1QzXbbrbSqTPKzsAVSzsn3H-S8e4dAaIIwqVFWa1d-bljp1cnh42b6b77rgB8BeVghY1My9VCmGie89vPx8nBdU3fl6sv-GZth0ttfLxhZVjkiTxZAyUJ0MaM3beKdVIbmmTNOJxoO6WfigZxQxT-pGBKkzL0GzFPcpclG6jYsRyCuht7vp2iDI33Q_FTro_cjqVywP_DVcsBPJy_KKPzhi2hzmTB1Dp7iwp0X6buk98Qt6jbEZqDYJKni6xXGrcJ49E0ulg7Omyh6CTn1zwQegDMovsw_4N-w5az8QHnow3stDa0SMzrp1PqZl1848PdQFRouAzuf7eRL2J6Svwdj8Q9-zVfgowBoEdLK-uyiPYKSoAxXacJcRaAb4VkdxsMNVE07OV87vwdOJ9R6ZrF2y3Tgjw7UVretIzGx-PGv4-HizHhUVn6lCajZPBvz3Lkj1F9xCxCRn-J0FRjgJQr47s999iLBcTYuzF4wm8M4cbB-xUuM4vY76WxUSKYJd5-laAHV4iUH91_QYiTDqx9HKygUMkZunYGB0Sv-QsDMDtyqBcCiaohxIVUAq-Buvg1LChRuNrHaIhFMr55z9qwVT0mNDf2-72qK_3_ADyUCuHKdrgaSYZGJTwhC47cZhBK2KxDFsOGXVnd-YN6qUSa9ijIp-w3PBQuk7-bhFJbN-6v5rxjtUapaHuEiK7cXTQFPPRt4gnSo6IyDOr_4Pp2xPNQZMWGw5szOHl1yCsYEo9FiYpE3IuyJ71BciraiQ9Cmr45fSrA1-wrxncr7ZPue09toI-9k9y0zTgXOqGoi1cHm7E06zZJ1WjIqolAlHTbfC-eVRWaLJ-HdHeh2ZP5dRdFUnNFeiwMcV0XXFFJLMmHOIUdy3Tt9zvfC35Lq2gDwxNLb1O1MLt6xDDJLpPBWDRj0dJWTTHDi0-ffIl6nf9GjvViEkekXpS3RbpFmkRWYj9rlEHeGRFif1sE7lfU0q1UtuymPrZA-u0eQoDAxmVE3USdGDVtJYJVfOHeuPaHB6DJkTRi3tm-XIhB418ZB5h3MfWwvHX4gMS7lAjf6ii6SYrGrXTcxib48ZfdWz08OGk4dpg04PMDtQNggmfwyLK6UpGtNiFjlUS6x7UblpqDOYLWUnC5ErdDABxlX3aioir3f6eOTtr3cLVQlx_zIkWoydxa9zV6hIiosz6KIJ1eAeLSQ0vAtZCWIJLhv_H9_A3xEq5YQgNLIt7dlIxWnjMlUjvTJXuWWBjfVLOifRmTOd2wimA_YjoAL4mY2iY2YndzOu3748re7PNg_QXfWQArd89FqLPkxdOHW19sl-xmcer8N4LRM2h14p6ZmJhVb_BdhyKtE3clvTpjYvs_y1KbHuwCDmwZ05TVqJ3dklnEPjDSL_KaOgBtR5DoBMc2Kmolakz0ogPLCi4hPgXpEOj6JYRuHDXsTj75buZM7zUv0aF_C2OqUBYFuLORRZlQBNATMExmMZcHyIm3qgktC7ZITeo8f-eI85q-RKHLg5ES6aIIspeqtOTESlFfnqV6R0rYTIlV9Du6_Egnixz_H15QdbxMNanONRzYy_RB6Khc2T-lc4Ndr4rnYp43_-1y6gJltIPqIVTTxXlFKW5Pixh-LhgQdV7jFEctEcoF5xW4Ck8sGf2L7bqxn2yI9HJnVMpPwT12q0nMV2dOEfYij1nDp6W-o5zZcpjyCiH6KyEc8UTYIfzmAVBTgFkX_Fd8tpICXHG66xWyX3UDJbMXaNVI-nPZSuCAsfBZ3h5upE_PPgIRlXNc7BxxFmM3HdPgvyHjiQGRL732muqYgrM2NP0jivxd9XsY1wcJGRBlK7MnwBecqvcNme-RN0oc1LQj-lzeCEP37RCyrNZDRY8QjeJT3O6Aj_NBEuzHHHFjyjxdC6OBuQxfOFm8RWHKNZWc36AuefsjliwzOyAG4dpdiZ-UYwV4E6k4ln8-I1YoxRfsOlqPTQ7YQc-uIz09R9MpQcZl9yGIs7J7a2i2DSUIHDkTSJbAw-5-G4zY", + "id": "", + "type": "compaction" + } + ], + "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", + "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 213, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 346 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt new file mode 100644 index 000000000..6b490a126 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -0,0 +1,1142 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "Sure! What would you like to know or discuss?", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 12, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-parse-operation + │ metadata: { + │ "operation": "responses-parse", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 20 + 4?" + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 23, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 69 + │ } + └── openai-responses-compact-operation + metadata: { + "operation": "responses-compact", + "testRunId": "" + } + └── openai.responses.compact [llm] + input: [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "id": "", + "role": "user", + "status": "completed", + "type": "message" + }, + { + "encrypted_content": "gAAAAABqBwV0tsDVAxoc2eIrNCzqFQMDcx_WtLB7jn1xjJshXoJiwnwd5Cu4dhaqCh9Xcc-MmsfZwlulYwKvJyc_9GMFb4a3hoZc-2P_IVhiMB3WBin89wlcd1L45sLcJ96FLirt0QD85Qq06DfX4ulYI_NiTA4jBtV2gZ-4EfubN4C8DrHHOvaQeq9twx6qNCEXCDOnJCeVC1QiblKYlQQmW4LEs8ej6QDonbxgwRZty2GxIXWXIzorpB9BD-JBlz9NruYz1gVJibftE3ItME06-Ej6ZTaGXvIYejuba9GWoIRVp4M74mBuWrD1cuvHHBOzLtOK0DD4VfuO7rYJkhrydbtBBEiG6KgmxuwAFNNcjUh8hlpdMi407ROMUhU4GhRq_t8lOJwTQmetb-U35B_DJqIiF5H2ub4_MNTfp732hqEUg51dAl4o4Ms75IMkApTgqwl1C_CnE_aIDkYtsGlVZo97M_2zS5DZ_aTzm2KrqKOM_fS68WZE4XZR8f5LaZyMnN5avHvl_dxE-tTsl_8SeyaLn3rj8mfEetv-Kf_Odld0fsNebcgakB8cZQ4tj-55guNDw3qqNoSXBCbnytk3Mr_WgdhzMofhZ3UrA6Vm1c5eyTtlX4GMwV09wgj-AcP48vfoOGJj-qNurjsGtdn30FzRlBUdGXExReHS0A2nr2-X3zcCkcjbEMoXjcHljpLbeuzkNRb6e5UyAQvQ0aIqa084U92k4icTyhlcHYPA2cVAap_P5c3qkkyZsiYBg13lzdKAkwMzpo6oznapLrhfwrvGIIW6k3dYEqMy5jfIZnrYz6BWWFuV2E9Iwg6R1tX8v3u2lG062YHgWSdT7uHk-ut68J-ilX92jbeWpy_1ML-sc866lstqW5qR_ogotAUWlIk8Xsw_ahN2HcGjVJuXzgwVU9QoaW8pwCimBfGF-dcP2-GvxTxBgGZSJe7iwA5wwFGEVgNE9Xrd3ESsTTTJyG_WLoAiBbZwWkIn68BYQzxJIWCJgXIF9-8BrGR5bD075IhFcMi9XTBdj3uzVjaDxv1t7_opxMXkxisZmCNxFcYuQZgOdWeRh07GtknWnDKJnPriVh17Rg1o8Yz8zGBHhoF2qgpmgwah7Gh1QzXbbrbSqTPKzsAVSzsn3H-S8e4dAaIIwqVFWa1d-bljp1cnh42b6b77rgB8BeVghY1My9VCmGie89vPx8nBdU3fl6sv-GZth0ttfLxhZVjkiTxZAyUJ0MaM3beKdVIbmmTNOJxoO6WfigZxQxT-pGBKkzL0GzFPcpclG6jYsRyCuht7vp2iDI33Q_FTro_cjqVywP_DVcsBPJy_KKPzhi2hzmTB1Dp7iwp0X6buk98Qt6jbEZqDYJKni6xXGrcJ49E0ulg7Omyh6CTn1zwQegDMovsw_4N-w5az8QHnow3stDa0SMzrp1PqZl1848PdQFRouAzuf7eRL2J6Svwdj8Q9-zVfgowBoEdLK-uyiPYKSoAxXacJcRaAb4VkdxsMNVE07OV87vwdOJ9R6ZrF2y3Tgjw7UVretIzGx-PGv4-HizHhUVn6lCajZPBvz3Lkj1F9xCxCRn-J0FRjgJQr47s999iLBcTYuzF4wm8M4cbB-xUuM4vY76WxUSKYJd5-laAHV4iUH91_QYiTDqx9HKygUMkZunYGB0Sv-QsDMDtyqBcCiaohxIVUAq-Buvg1LChRuNrHaIhFMr55z9qwVT0mNDf2-72qK_3_ADyUCuHKdrgaSYZGJTwhC47cZhBK2KxDFsOGXVnd-YN6qUSa9ijIp-w3PBQuk7-bhFJbN-6v5rxjtUapaHuEiK7cXTQFPPRt4gnSo6IyDOr_4Pp2xPNQZMWGw5szOHl1yCsYEo9FiYpE3IuyJ71BciraiQ9Cmr45fSrA1-wrxncr7ZPue09toI-9k9y0zTgXOqGoi1cHm7E06zZJ1WjIqolAlHTbfC-eVRWaLJ-HdHeh2ZP5dRdFUnNFeiwMcV0XXFFJLMmHOIUdy3Tt9zvfC35Lq2gDwxNLb1O1MLt6xDDJLpPBWDRj0dJWTTHDi0-ffIl6nf9GjvViEkekXpS3RbpFmkRWYj9rlEHeGRFif1sE7lfU0q1UtuymPrZA-u0eQoDAxmVE3USdGDVtJYJVfOHeuPaHB6DJkTRi3tm-XIhB418ZB5h3MfWwvHX4gMS7lAjf6ii6SYrGrXTcxib48ZfdWz08OGk4dpg04PMDtQNggmfwyLK6UpGtNiFjlUS6x7UblpqDOYLWUnC5ErdDABxlX3aioir3f6eOTtr3cLVQlx_zIkWoydxa9zV6hIiosz6KIJ1eAeLSQ0vAtZCWIJLhv_H9_A3xEq5YQgNLIt7dlIxWnjMlUjvTJXuWWBjfVLOifRmTOd2wimA_YjoAL4mY2iY2YndzOu3748re7PNg_QXfWQArd89FqLPkxdOHW19sl-xmcer8N4LRM2h14p6ZmJhVb_BdhyKtE3clvTpjYvs_y1KbHuwCDmwZ05TVqJ3dklnEPjDSL_KaOgBtR5DoBMc2Kmolakz0ogPLCi4hPgXpEOj6JYRuHDXsTj75buZM7zUv0aF_C2OqUBYFuLORRZlQBNATMExmMZcHyIm3qgktC7ZITeo8f-eI85q-RKHLg5ES6aIIspeqtOTESlFfnqV6R0rYTIlV9Du6_Egnixz_H15QdbxMNanONRzYy_RB6Khc2T-lc4Ndr4rnYp43_-1y6gJltIPqIVTTxXlFKW5Pixh-LhgQdV7jFEctEcoF5xW4Ck8sGf2L7bqxn2yI9HJnVMpPwT12q0nMV2dOEfYij1nDp6W-o5zZcpjyCiH6KyEc8UTYIfzmAVBTgFkX_Fd8tpICXHG66xWyX3UDJbMXaNVI-nPZSuCAsfBZ3h5upE_PPgIRlXNc7BxxFmM3HdPgvyHjiQGRL732muqYgrM2NP0jivxd9XsY1wcJGRBlK7MnwBecqvcNme-RN0oc1LQj-lzeCEP37RCyrNZDRY8QjeJT3O6Aj_NBEuzHHHFjyjxdC6OBuQxfOFm8RWHKNZWc36AuefsjliwzOyAG4dpdiZ-UYwV4E6k4ln8-I1YoxRfsOlqPTQ7YQc-uIz09R9MpQcZl9yGIs7J7a2i2DSUIHDkTSJbAw-5-G4zY", + "id": "", + "type": "compaction" + } + ] + metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", + "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", + "provider": "openai" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 213, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 346 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json new file mode 100644 index 000000000..36d504ea1 --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -0,0 +1,1146 @@ +{ + "span_tree": [ + { + "name": "openai-instrumentation-root", + "type": "task", + "children": [ + { + "name": "openai-client-private-fields-operation", + "children": [], + "metadata": { + "operation": "client-private-fields", + "testRunId": "" + } + }, + { + "name": "openai-chat-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "OK.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openai-chat-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly FOUR.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "Four.", + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "openai-stream-with-response-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM-WITH-RESPONSE.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM-WITH-RESPONSE", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 6, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "stream-with-response", + "testRunId": "" + } + }, + { + "name": "openai-stream-fixture-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with a refusal stream fixture.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO", + "top_logprobs": [ + { + "bytes": [ + 78, + 79 + ], + "logprob": -0.1, + "token": "NO" + } + ] + }, + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE", + "top_logprobs": [ + { + "bytes": [ + 80, + 69 + ], + "logprob": -0.2, + "token": "PE" + } + ] + } + ] + }, + "message": { + "refusal": "NOPE", + "role": "assistant" + } + } + ], + "metadata": { + "logprobs": true, + "max_tokens": 12, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "temperature": 0, + "top_logprobs": 2 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "stream-fixture", + "testRunId": "" + } + }, + { + "name": "openai-parse-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "What is 2 + 2?", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "annotations": [], + "content": "{\"answer\":4}", + "parsed": { + "answer": 4 + }, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "response_format": { + "json_schema": { + "name": "math_response", + "schema": { + "properties": { + "answer": { + "type": "number" + } + }, + "required": [ + "answer" + ], + "type": "object" + } + }, + "type": "json_schema" + } + }, + "metrics": { + "completion_accepted_prediction_tokens": 0, + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_rejected_prediction_tokens": 0, + "completion_tokens": 5, + "prompt_audio_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 45, + "time_to_first_token": 0, + "tokens": 50 + } + } + ], + "metadata": { + "operation": "parse", + "testRunId": "" + } + }, + { + "name": "openai-sync-stream-operation", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly SYNC STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "SYNC STREAM", + "parsed": null, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "time_to_first_token": 0 + } + } + ], + "metadata": { + "operation": "sync-stream", + "testRunId": "" + } + }, + { + "name": "openai-embeddings-operation", + "children": [ + { + "name": "Embedding", + "type": "llm", + "children": [], + "input": "Paris", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "model": "text-embedding-3-small", + "provider": "openai" + }, + "metrics": { + "prompt_tokens": 1, + "tokens": 1 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openai-moderations-operation", + "children": [ + { + "name": "Moderation", + "type": "llm", + "children": [], + "input": "Hello from Braintrust.", + "output": [ + { + "categories": { + "harassment": false, + "harassment/threatening": false, + "hate": false, + "hate/threatening": false, + "illicit": false, + "illicit/violent": false, + "self-harm": false, + "self-harm/instructions": false, + "self-harm/intent": false, + "sexual": false, + "sexual/minors": false, + "violence": false, + "violence/graphic": false + }, + "category_applied_input_types": { + "harassment": [ + "text" + ], + "harassment/threatening": [ + "text" + ], + "hate": [ + "text" + ], + "hate/threatening": [ + "text" + ], + "illicit": [ + "text" + ], + "illicit/violent": [ + "text" + ], + "self-harm": [ + "text" + ], + "self-harm/instructions": [ + "text" + ], + "self-harm/intent": [ + "text" + ], + "sexual": [ + "text" + ], + "sexual/minors": [ + "text" + ], + "violence": [ + "text" + ], + "violence/graphic": [ + "text" + ] + }, + "category_scores": { + "harassment": 0.00001141223854434603, + "harassment/threatening": 0.0000012219076020491008, + "hate": 0.000003944258675190877, + "hate/threatening": 1.2878952156558146e-7, + "illicit": 0.000005738759521437678, + "illicit/violent": 0.000003822911232549001, + "self-harm": 0.000003120191139396651, + "self-harm/instructions": 6.240930669504435e-7, + "self-harm/intent": 0.0000022474089854291757, + "sexual": 0.000006814872211615988, + "sexual/minors": 5.255395707074638e-7, + "violence": 0.000014202364022997911, + "violence/graphic": 0.0000019525885208642222 + }, + "flagged": false + } + ], + "metadata": { + "model": "omni-moderation-2024-09-26", + "provider": "openai" + } + } + ], + "metadata": { + "operation": "moderations", + "testRunId": "" + } + }, + { + "name": "openai-responses-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARIS.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARIS", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "PARIS", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openai-responses-with-response-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 2 + 2? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "4", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_text": "4", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-with-response", + "testRunId": "" + } + }, + { + "name": "openai-responses-create-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly RESPONSE STREAM.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "Sure! What would you like to know or discuss?", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "stream": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 12, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 25 + } + } + ], + "metadata": { + "operation": "responses-create-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "What is 6 x 6? Reply with just the number.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "36", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 21, + "time_to_first_token": 0, + "tokens": 23 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openai-responses-stream-partial-operation", + "children": [ + { + "name": "openai.responses.create", + "type": "llm", + "children": [], + "input": "Reply with exactly PARTIAL.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "PARTIAL.", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": 24, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 4, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-stream-partial", + "testRunId": "" + } + }, + { + "name": "openai-responses-parse-operation", + "children": [ + { + "name": "openai.responses.parse", + "type": "llm", + "children": [], + "input": "What is 20 + 4?", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "parsed": { + "reasoning": "Adding 20 and 4 gives you a total of 24.", + "value": 24 + }, + "text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 0, + "created_at": 0, + "error": null, + "frequency_penalty": 0, + "id": "", + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "moderation": null, + "object": "response", + "output_parsed": { + "reasoning": "Adding 20 and 4 gives you a total of 24.", + "value": 24 + }, + "output_text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": "[REDACTED]", + "prompt_cache_retention": "in_memory", + "provider": "openai", + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "status": "completed", + "store": true, + "temperature": 1, + "text": { + "format": { + "description": null, + "name": "NumberAnswer", + "schema": { + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "value", + "reasoning" + ], + "type": "object" + }, + "strict": true, + "type": "json_schema" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1, + "truncation": "disabled", + "user": null + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 23, + "prompt_cached_tokens": 0, + "prompt_tokens": 46, + "time_to_first_token": 0, + "tokens": 69 + } + } + ], + "metadata": { + "operation": "responses-parse", + "testRunId": "" + } + }, + { + "name": "openai-responses-compact-operation", + "children": [ + { + "name": "openai.responses.compact", + "type": "llm", + "children": [], + "input": [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], + "role": "assistant" + } + ], + "output": [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "id": "", + "role": "user", + "status": "completed", + "type": "message" + }, + { + "encrypted_content": "gAAAAABqBwV0tsDVAxoc2eIrNCzqFQMDcx_WtLB7jn1xjJshXoJiwnwd5Cu4dhaqCh9Xcc-MmsfZwlulYwKvJyc_9GMFb4a3hoZc-2P_IVhiMB3WBin89wlcd1L45sLcJ96FLirt0QD85Qq06DfX4ulYI_NiTA4jBtV2gZ-4EfubN4C8DrHHOvaQeq9twx6qNCEXCDOnJCeVC1QiblKYlQQmW4LEs8ej6QDonbxgwRZty2GxIXWXIzorpB9BD-JBlz9NruYz1gVJibftE3ItME06-Ej6ZTaGXvIYejuba9GWoIRVp4M74mBuWrD1cuvHHBOzLtOK0DD4VfuO7rYJkhrydbtBBEiG6KgmxuwAFNNcjUh8hlpdMi407ROMUhU4GhRq_t8lOJwTQmetb-U35B_DJqIiF5H2ub4_MNTfp732hqEUg51dAl4o4Ms75IMkApTgqwl1C_CnE_aIDkYtsGlVZo97M_2zS5DZ_aTzm2KrqKOM_fS68WZE4XZR8f5LaZyMnN5avHvl_dxE-tTsl_8SeyaLn3rj8mfEetv-Kf_Odld0fsNebcgakB8cZQ4tj-55guNDw3qqNoSXBCbnytk3Mr_WgdhzMofhZ3UrA6Vm1c5eyTtlX4GMwV09wgj-AcP48vfoOGJj-qNurjsGtdn30FzRlBUdGXExReHS0A2nr2-X3zcCkcjbEMoXjcHljpLbeuzkNRb6e5UyAQvQ0aIqa084U92k4icTyhlcHYPA2cVAap_P5c3qkkyZsiYBg13lzdKAkwMzpo6oznapLrhfwrvGIIW6k3dYEqMy5jfIZnrYz6BWWFuV2E9Iwg6R1tX8v3u2lG062YHgWSdT7uHk-ut68J-ilX92jbeWpy_1ML-sc866lstqW5qR_ogotAUWlIk8Xsw_ahN2HcGjVJuXzgwVU9QoaW8pwCimBfGF-dcP2-GvxTxBgGZSJe7iwA5wwFGEVgNE9Xrd3ESsTTTJyG_WLoAiBbZwWkIn68BYQzxJIWCJgXIF9-8BrGR5bD075IhFcMi9XTBdj3uzVjaDxv1t7_opxMXkxisZmCNxFcYuQZgOdWeRh07GtknWnDKJnPriVh17Rg1o8Yz8zGBHhoF2qgpmgwah7Gh1QzXbbrbSqTPKzsAVSzsn3H-S8e4dAaIIwqVFWa1d-bljp1cnh42b6b77rgB8BeVghY1My9VCmGie89vPx8nBdU3fl6sv-GZth0ttfLxhZVjkiTxZAyUJ0MaM3beKdVIbmmTNOJxoO6WfigZxQxT-pGBKkzL0GzFPcpclG6jYsRyCuht7vp2iDI33Q_FTro_cjqVywP_DVcsBPJy_KKPzhi2hzmTB1Dp7iwp0X6buk98Qt6jbEZqDYJKni6xXGrcJ49E0ulg7Omyh6CTn1zwQegDMovsw_4N-w5az8QHnow3stDa0SMzrp1PqZl1848PdQFRouAzuf7eRL2J6Svwdj8Q9-zVfgowBoEdLK-uyiPYKSoAxXacJcRaAb4VkdxsMNVE07OV87vwdOJ9R6ZrF2y3Tgjw7UVretIzGx-PGv4-HizHhUVn6lCajZPBvz3Lkj1F9xCxCRn-J0FRjgJQr47s999iLBcTYuzF4wm8M4cbB-xUuM4vY76WxUSKYJd5-laAHV4iUH91_QYiTDqx9HKygUMkZunYGB0Sv-QsDMDtyqBcCiaohxIVUAq-Buvg1LChRuNrHaIhFMr55z9qwVT0mNDf2-72qK_3_ADyUCuHKdrgaSYZGJTwhC47cZhBK2KxDFsOGXVnd-YN6qUSa9ijIp-w3PBQuk7-bhFJbN-6v5rxjtUapaHuEiK7cXTQFPPRt4gnSo6IyDOr_4Pp2xPNQZMWGw5szOHl1yCsYEo9FiYpE3IuyJ71BciraiQ9Cmr45fSrA1-wrxncr7ZPue09toI-9k9y0zTgXOqGoi1cHm7E06zZJ1WjIqolAlHTbfC-eVRWaLJ-HdHeh2ZP5dRdFUnNFeiwMcV0XXFFJLMmHOIUdy3Tt9zvfC35Lq2gDwxNLb1O1MLt6xDDJLpPBWDRj0dJWTTHDi0-ffIl6nf9GjvViEkekXpS3RbpFmkRWYj9rlEHeGRFif1sE7lfU0q1UtuymPrZA-u0eQoDAxmVE3USdGDVtJYJVfOHeuPaHB6DJkTRi3tm-XIhB418ZB5h3MfWwvHX4gMS7lAjf6ii6SYrGrXTcxib48ZfdWz08OGk4dpg04PMDtQNggmfwyLK6UpGtNiFjlUS6x7UblpqDOYLWUnC5ErdDABxlX3aioir3f6eOTtr3cLVQlx_zIkWoydxa9zV6hIiosz6KIJ1eAeLSQ0vAtZCWIJLhv_H9_A3xEq5YQgNLIt7dlIxWnjMlUjvTJXuWWBjfVLOifRmTOd2wimA_YjoAL4mY2iY2YndzOu3748re7PNg_QXfWQArd89FqLPkxdOHW19sl-xmcer8N4LRM2h14p6ZmJhVb_BdhyKtE3clvTpjYvs_y1KbHuwCDmwZ05TVqJ3dklnEPjDSL_KaOgBtR5DoBMc2Kmolakz0ogPLCi4hPgXpEOj6JYRuHDXsTj75buZM7zUv0aF_C2OqUBYFuLORRZlQBNATMExmMZcHyIm3qgktC7ZITeo8f-eI85q-RKHLg5ES6aIIspeqtOTESlFfnqV6R0rYTIlV9Du6_Egnixz_H15QdbxMNanONRzYy_RB6Khc2T-lc4Ndr4rnYp43_-1y6gJltIPqIVTTxXlFKW5Pixh-LhgQdV7jFEctEcoF5xW4Ck8sGf2L7bqxn2yI9HJnVMpPwT12q0nMV2dOEfYij1nDp6W-o5zZcpjyCiH6KyEc8UTYIfzmAVBTgFkX_Fd8tpICXHG66xWyX3UDJbMXaNVI-nPZSuCAsfBZ3h5upE_PPgIRlXNc7BxxFmM3HdPgvyHjiQGRL732muqYgrM2NP0jivxd9XsY1wcJGRBlK7MnwBecqvcNme-RN0oc1LQj-lzeCEP37RCyrNZDRY8QjeJT3O6Aj_NBEuzHHHFjyjxdC6OBuQxfOFm8RWHKNZWc36AuefsjliwzOyAG4dpdiZ-UYwV4E6k4ln8-I1YoxRfsOlqPTQ7YQc-uIz09R9MpQcZl9yGIs7J7a2i2DSUIHDkTSJbAw-5-G4zY", + "id": "", + "type": "compaction" + } + ], + "metadata": { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", + "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", + "provider": "openai" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 213, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 346 + } + } + ], + "metadata": { + "operation": "responses-compact", + "testRunId": "" + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt new file mode 100644 index 000000000..e8864ecdf --- /dev/null +++ b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -0,0 +1,1007 @@ +span_tree: +└── openai-instrumentation-root [task] + metadata: { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-instrumentation", + "testRunId": "" + } + ├── openai-client-private-fields-operation + │ metadata: { + │ "operation": "client-private-fields", + │ "testRunId": "" + │ } + ├── openai-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "OK.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-chat-with-response-operation + │ metadata: { + │ "operation": "chat-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly FOUR.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "Four.", + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openai-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openai-stream-with-response-operation + │ metadata: { + │ "operation": "stream-with-response", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM-WITH-RESPONSE.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM-WITH-RESPONSE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 6, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-stream-fixture-operation + │ metadata: { + │ "operation": "stream-fixture", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with a refusal stream fixture.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": { + │ "content": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 78, + │ 79 + │ ], + │ "logprob": -0.1, + │ "token": "NO" + │ } + │ ] + │ }, + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE", + │ "top_logprobs": [ + │ { + │ "bytes": [ + │ 80, + │ 69 + │ ], + │ "logprob": -0.2, + │ "token": "PE" + │ } + │ ] + │ } + │ ] + │ }, + │ "message": { + │ "refusal": "NOPE", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "logprobs": true, + │ "max_tokens": 12, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "temperature": 0, + │ "top_logprobs": 2 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-parse-operation + │ metadata: { + │ "operation": "parse", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "What is 2 + 2?", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "annotations": [], + │ "content": "{\"answer\":4}", + │ "parsed": { + │ "answer": 4 + │ }, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "response_format": { + │ "json_schema": { + │ "name": "math_response", + │ "schema": { + │ "properties": { + │ "answer": { + │ "type": "number" + │ } + │ }, + │ "required": [ + │ "answer" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "json_schema" + │ } + │ } + │ metrics: { + │ "completion_accepted_prediction_tokens": 0, + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_rejected_prediction_tokens": 0, + │ "completion_tokens": 5, + │ "prompt_audio_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 45, + │ "time_to_first_token": 0, + │ "tokens": 50 + │ } + ├── openai-sync-stream-operation + │ metadata: { + │ "operation": "sync-stream", + │ "testRunId": "" + │ } + │ └── Chat Completion [llm] + │ input: [ + │ { + │ "content": "Reply with exactly SYNC STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "SYNC STREAM", + │ "parsed": null, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "time_to_first_token": 0 + │ } + ├── openai-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── Embedding [llm] + │ input: "Paris" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "model": "text-embedding-3-small", + │ "provider": "openai" + │ } + │ metrics: { + │ "prompt_tokens": 1, + │ "tokens": 1 + │ } + ├── openai-moderations-operation + │ metadata: { + │ "operation": "moderations", + │ "testRunId": "" + │ } + │ └── Moderation [llm] + │ input: "Hello from Braintrust." + │ output: [ + │ { + │ "categories": { + │ "harassment": false, + │ "harassment/threatening": false, + │ "hate": false, + │ "hate/threatening": false, + │ "illicit": false, + │ "illicit/violent": false, + │ "self-harm": false, + │ "self-harm/instructions": false, + │ "self-harm/intent": false, + │ "sexual": false, + │ "sexual/minors": false, + │ "violence": false, + │ "violence/graphic": false + │ }, + │ "category_applied_input_types": { + │ "harassment": [ + │ "text" + │ ], + │ "harassment/threatening": [ + │ "text" + │ ], + │ "hate": [ + │ "text" + │ ], + │ "hate/threatening": [ + │ "text" + │ ], + │ "illicit": [ + │ "text" + │ ], + │ "illicit/violent": [ + │ "text" + │ ], + │ "self-harm": [ + │ "text" + │ ], + │ "self-harm/instructions": [ + │ "text" + │ ], + │ "self-harm/intent": [ + │ "text" + │ ], + │ "sexual": [ + │ "text" + │ ], + │ "sexual/minors": [ + │ "text" + │ ], + │ "violence": [ + │ "text" + │ ], + │ "violence/graphic": [ + │ "text" + │ ] + │ }, + │ "category_scores": { + │ "harassment": 0.00001141223854434603, + │ "harassment/threatening": 0.0000012219076020491008, + │ "hate": 0.000003944258675190877, + │ "hate/threatening": 1.2878952156558146e-7, + │ "illicit": 0.000005738759521437678, + │ "illicit/violent": 0.000003822911232549001, + │ "self-harm": 0.000003120191139396651, + │ "self-harm/instructions": 6.240930669504435e-7, + │ "self-harm/intent": 0.0000022474089854291757, + │ "sexual": 0.000006814872211615988, + │ "sexual/minors": 5.255395707074638e-7, + │ "violence": 0.000014202364022997911, + │ "violence/graphic": 0.0000019525885208642222 + │ }, + │ "flagged": false + │ } + │ ] + │ metadata: { + │ "model": "omni-moderation-2024-09-26", + │ "provider": "openai" + │ } + ├── openai-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARIS." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARIS", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "PARIS", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 16 + │ } + ├── openai-responses-with-response-operation + │ metadata: { + │ "operation": "responses-with-response", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 2 + 2? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "4", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_text": "4", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-create-stream-operation + │ metadata: { + │ "operation": "responses-create-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly RESPONSE STREAM." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "Sure! What would you like to know or discuss?", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "stream": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 12, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 25 + │ } + ├── openai-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "What is 6 x 6? Reply with just the number." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "36", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 21, + │ "time_to_first_token": 0, + │ "tokens": 23 + │ } + ├── openai-responses-stream-partial-operation + │ metadata: { + │ "operation": "responses-stream-partial", + │ "testRunId": "" + │ } + │ └── openai.responses.create [llm] + │ input: "Reply with exactly PARTIAL." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "PARTIAL.", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": 24, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": null, + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "type": "text" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 4, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 13, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openai-responses-parse-operation + │ metadata: { + │ "operation": "responses-parse", + │ "testRunId": "" + │ } + │ └── openai.responses.parse [llm] + │ input: "What is 20 + 4?" + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "parsed": { + │ "reasoning": "Adding 20 and 4 gives you a total of 24.", + │ "value": 24 + │ }, + │ "text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "billing": { + │ "payer": "developer" + │ }, + │ "completed_at": 0, + │ "created_at": 0, + │ "error": null, + │ "frequency_penalty": 0, + │ "id": "", + │ "incomplete_details": null, + │ "instructions": null, + │ "max_output_tokens": null, + │ "max_tool_calls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "moderation": null, + │ "object": "response", + │ "output_parsed": { + │ "reasoning": "Adding 20 and 4 gives you a total of 24.", + │ "value": 24 + │ }, + │ "output_text": "{\"reasoning\":\"Adding 20 and 4 gives you a total of 24.\",\"value\":24}", + │ "parallel_tool_calls": true, + │ "presence_penalty": 0, + │ "previous_response_id": null, + │ "prompt_cache_key": "[REDACTED]", + │ "prompt_cache_retention": "in_memory", + │ "provider": "openai", + │ "reasoning": { + │ "effort": null, + │ "summary": null + │ }, + │ "safety_identifier": null, + │ "service_tier": "default", + │ "status": "completed", + │ "store": true, + │ "temperature": 1, + │ "text": { + │ "format": { + │ "description": null, + │ "name": "NumberAnswer", + │ "schema": { + │ "additionalProperties": false, + │ "properties": { + │ "reasoning": { + │ "type": "string" + │ }, + │ "value": { + │ "type": "integer" + │ } + │ }, + │ "required": [ + │ "value", + │ "reasoning" + │ ], + │ "type": "object" + │ }, + │ "strict": true, + │ "type": "json_schema" + │ }, + │ "verbosity": "medium" + │ }, + │ "tool_choice": "auto", + │ "tools": [], + │ "top_logprobs": 0, + │ "top_p": 1, + │ "truncation": "disabled", + │ "user": null + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 23, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 46, + │ "time_to_first_token": 0, + │ "tokens": 69 + │ } + └── openai-responses-compact-operation + metadata: { + "operation": "responses-compact", + "testRunId": "" + } + └── openai.responses.compact [llm] + input: [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "role": "user" + }, + { + "content": [ + { + "text": "Understood. I will keep answers concise.", + "type": "output_text" + } + ], + "role": "assistant" + } + ] + output: [ + { + "content": [ + { + "text": "I live in Paris and prefer concise answers.", + "type": "input_text" + } + ], + "id": "", + "role": "user", + "status": "completed", + "type": "message" + }, + { + "encrypted_content": "gAAAAABqBwV0tsDVAxoc2eIrNCzqFQMDcx_WtLB7jn1xjJshXoJiwnwd5Cu4dhaqCh9Xcc-MmsfZwlulYwKvJyc_9GMFb4a3hoZc-2P_IVhiMB3WBin89wlcd1L45sLcJ96FLirt0QD85Qq06DfX4ulYI_NiTA4jBtV2gZ-4EfubN4C8DrHHOvaQeq9twx6qNCEXCDOnJCeVC1QiblKYlQQmW4LEs8ej6QDonbxgwRZty2GxIXWXIzorpB9BD-JBlz9NruYz1gVJibftE3ItME06-Ej6ZTaGXvIYejuba9GWoIRVp4M74mBuWrD1cuvHHBOzLtOK0DD4VfuO7rYJkhrydbtBBEiG6KgmxuwAFNNcjUh8hlpdMi407ROMUhU4GhRq_t8lOJwTQmetb-U35B_DJqIiF5H2ub4_MNTfp732hqEUg51dAl4o4Ms75IMkApTgqwl1C_CnE_aIDkYtsGlVZo97M_2zS5DZ_aTzm2KrqKOM_fS68WZE4XZR8f5LaZyMnN5avHvl_dxE-tTsl_8SeyaLn3rj8mfEetv-Kf_Odld0fsNebcgakB8cZQ4tj-55guNDw3qqNoSXBCbnytk3Mr_WgdhzMofhZ3UrA6Vm1c5eyTtlX4GMwV09wgj-AcP48vfoOGJj-qNurjsGtdn30FzRlBUdGXExReHS0A2nr2-X3zcCkcjbEMoXjcHljpLbeuzkNRb6e5UyAQvQ0aIqa084U92k4icTyhlcHYPA2cVAap_P5c3qkkyZsiYBg13lzdKAkwMzpo6oznapLrhfwrvGIIW6k3dYEqMy5jfIZnrYz6BWWFuV2E9Iwg6R1tX8v3u2lG062YHgWSdT7uHk-ut68J-ilX92jbeWpy_1ML-sc866lstqW5qR_ogotAUWlIk8Xsw_ahN2HcGjVJuXzgwVU9QoaW8pwCimBfGF-dcP2-GvxTxBgGZSJe7iwA5wwFGEVgNE9Xrd3ESsTTTJyG_WLoAiBbZwWkIn68BYQzxJIWCJgXIF9-8BrGR5bD075IhFcMi9XTBdj3uzVjaDxv1t7_opxMXkxisZmCNxFcYuQZgOdWeRh07GtknWnDKJnPriVh17Rg1o8Yz8zGBHhoF2qgpmgwah7Gh1QzXbbrbSqTPKzsAVSzsn3H-S8e4dAaIIwqVFWa1d-bljp1cnh42b6b77rgB8BeVghY1My9VCmGie89vPx8nBdU3fl6sv-GZth0ttfLxhZVjkiTxZAyUJ0MaM3beKdVIbmmTNOJxoO6WfigZxQxT-pGBKkzL0GzFPcpclG6jYsRyCuht7vp2iDI33Q_FTro_cjqVywP_DVcsBPJy_KKPzhi2hzmTB1Dp7iwp0X6buk98Qt6jbEZqDYJKni6xXGrcJ49E0ulg7Omyh6CTn1zwQegDMovsw_4N-w5az8QHnow3stDa0SMzrp1PqZl1848PdQFRouAzuf7eRL2J6Svwdj8Q9-zVfgowBoEdLK-uyiPYKSoAxXacJcRaAb4VkdxsMNVE07OV87vwdOJ9R6ZrF2y3Tgjw7UVretIzGx-PGv4-HizHhUVn6lCajZPBvz3Lkj1F9xCxCRn-J0FRjgJQr47s999iLBcTYuzF4wm8M4cbB-xUuM4vY76WxUSKYJd5-laAHV4iUH91_QYiTDqx9HKygUMkZunYGB0Sv-QsDMDtyqBcCiaohxIVUAq-Buvg1LChRuNrHaIhFMr55z9qwVT0mNDf2-72qK_3_ADyUCuHKdrgaSYZGJTwhC47cZhBK2KxDFsOGXVnd-YN6qUSa9ijIp-w3PBQuk7-bhFJbN-6v5rxjtUapaHuEiK7cXTQFPPRt4gnSo6IyDOr_4Pp2xPNQZMWGw5szOHl1yCsYEo9FiYpE3IuyJ71BciraiQ9Cmr45fSrA1-wrxncr7ZPue09toI-9k9y0zTgXOqGoi1cHm7E06zZJ1WjIqolAlHTbfC-eVRWaLJ-HdHeh2ZP5dRdFUnNFeiwMcV0XXFFJLMmHOIUdy3Tt9zvfC35Lq2gDwxNLb1O1MLt6xDDJLpPBWDRj0dJWTTHDi0-ffIl6nf9GjvViEkekXpS3RbpFmkRWYj9rlEHeGRFif1sE7lfU0q1UtuymPrZA-u0eQoDAxmVE3USdGDVtJYJVfOHeuPaHB6DJkTRi3tm-XIhB418ZB5h3MfWwvHX4gMS7lAjf6ii6SYrGrXTcxib48ZfdWz08OGk4dpg04PMDtQNggmfwyLK6UpGtNiFjlUS6x7UblpqDOYLWUnC5ErdDABxlX3aioir3f6eOTtr3cLVQlx_zIkWoydxa9zV6hIiosz6KIJ1eAeLSQ0vAtZCWIJLhv_H9_A3xEq5YQgNLIt7dlIxWnjMlUjvTJXuWWBjfVLOifRmTOd2wimA_YjoAL4mY2iY2YndzOu3748re7PNg_QXfWQArd89FqLPkxdOHW19sl-xmcer8N4LRM2h14p6ZmJhVb_BdhyKtE3clvTpjYvs_y1KbHuwCDmwZ05TVqJ3dklnEPjDSL_KaOgBtR5DoBMc2Kmolakz0ogPLCi4hPgXpEOj6JYRuHDXsTj75buZM7zUv0aF_C2OqUBYFuLORRZlQBNATMExmMZcHyIm3qgktC7ZITeo8f-eI85q-RKHLg5ES6aIIspeqtOTESlFfnqV6R0rYTIlV9Du6_Egnixz_H15QdbxMNanONRzYy_RB6Khc2T-lc4Ndr4rnYp43_-1y6gJltIPqIVTTxXlFKW5Pixh-LhgQdV7jFEctEcoF5xW4Ck8sGf2L7bqxn2yI9HJnVMpPwT12q0nMV2dOEfYij1nDp6W-o5zZcpjyCiH6KyEc8UTYIfzmAVBTgFkX_Fd8tpICXHG66xWyX3UDJbMXaNVI-nPZSuCAsfBZ3h5upE_PPgIRlXNc7BxxFmM3HdPgvyHjiQGRL732muqYgrM2NP0jivxd9XsY1wcJGRBlK7MnwBecqvcNme-RN0oc1LQj-lzeCEP37RCyrNZDRY8QjeJT3O6Aj_NBEuzHHHFjyjxdC6OBuQxfOFm8RWHKNZWc36AuefsjliwzOyAG4dpdiZ-UYwV4E6k4ln8-I1YoxRfsOlqPTQ7YQc-uIz09R9MpQcZl9yGIs7J7a2i2DSUIHDkTSJbAw-5-G4zY", + "id": "", + "type": "compaction" + } + ] + metadata: { + "created_at": 0, + "id": "", + "instructions": "Preserve only durable user preferences.", + "model": "gpt-4o-mini-2024-07-18", + "object": "response.compaction", + "provider": "openai" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 213, + "prompt_cached_tokens": 0, + "prompt_tokens": 133, + "time_to_first_token": 0, + "tokens": 346 + } diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json deleted file mode 100644 index 03d4374b8..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.log-payloads.json +++ /dev/null @@ -1,631 +0,0 @@ -[ - { - "input": { - "kind": "undefined" - }, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "output": null, - "type": "task" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [ - "answer" - ], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "string", - "role": "user" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "output": [ - { - "json_keys": [], - "role": "assistant" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "output": { - "embedding_length": 1536 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "output": { - "flagged_count": 0, - "result_count": 1 - }, - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "output": null, - "type": null - }, - { - "input": { - "kind": "text" - }, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "output": [ - { - "content_types": [ - "output_text" - ], - "json_keys": [ - "reasoning", - "value" - ], - "role": "assistant", - "status": "completed", - "type": "message" - } - ], - "type": "llm" - }, - { - "input": { - "kind": "undefined" - }, - "metadata": { - "operation": "responses-compact" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-compact-operation", - "output": null, - "type": null - }, - { - "input": [ - { - "content_kind": "blocks", - "role": "user" - }, - { - "content_kind": "blocks", - "role": "assistant" - } - ], - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.compact", - "output": [ - { - "content_types": [ - "input_text" - ], - "json_keys": [], - "role": "user", - "status": "completed", - "type": "message" - }, - { - "content_types": [], - "json_keys": [], - "role": null, - "status": null, - "type": "compaction" - } - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json b/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json deleted file mode 100644 index 483d487d1..000000000 --- a/e2e/scenarios/openai-instrumentation/__snapshots__/openai-v6.span-events.json +++ /dev/null @@ -1,415 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "openaiSdkVersion": "", - "scenario": "openai-instrumentation" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-instrumentation-root", - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-chat-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream-fixture" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-stream-fixture-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "sync-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-sync-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "Chat Completion", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-embeddings-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Embedding", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "moderations" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-moderations-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "omni-moderation-2024-09-26", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "Moderation", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-with-response" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-with-response-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-create-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-create-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream-partial" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-stream-partial-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.create", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-parse" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-parse-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.parse", - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-compact" - }, - "metrics": { - "has_time_to_first_token": false - }, - "name": "openai-responses-compact-operation", - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metrics": { - "has_time_to_first_token": true - }, - "name": "openai.responses.compact", - "type": "llm" - } -] diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index d5fe27c0b..43fa175d4 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -2,17 +2,18 @@ import { existsSync } from "node:fs"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { + matchSpanTreeSnapshot, + spanTreeFields, + type SpanTreeEntry, +} from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -462,23 +463,6 @@ function summarizeOutput(name: string, output: Json): Json { : ({ kind: typeof output } satisfies Json); } -function summarizeOpenAISpan( - event: CapturedLogEvent, - summaryName: string | undefined, -): Json { - return { - has_input: event.input !== undefined && event.input !== null, - has_output: event.output !== undefined && event.output !== null, - metadata: pickMetadata( - event.row.metadata as Record | undefined, - ["model", "openaiSdkVersion", "operation", "provider", "scenario"], - ), - metrics: summarizeMetricPresence(event.metrics as Json), - name: summaryName ?? event.span.name ?? null, - type: event.span.type ?? null, - } satisfies Json; -} - function summarizeOpenAIPayload( event: CapturedLogEvent, summaryName: string | undefined, @@ -533,30 +517,36 @@ function buildRelevantEvents( return relevantEvents; } -function buildSpanSummary( +function buildSpanTree( events: CapturedLogEvent[], operationSpecs: OperationSpec[], -): Json { - return normalizeForSnapshot( - buildRelevantEvents(events, operationSpecs).map(({ event, summaryName }) => - summarizeOpenAISpan(event, summaryName), - ) as Json, - ); -} - -function buildPayloadSummary( - events: CapturedLogEvent[], - operationSpecs: OperationSpec[], -): Json { - return normalizeForSnapshot( - buildRelevantEvents(events, operationSpecs).map(({ event, summaryName }) => - summarizeOpenAIPayload(event, summaryName), - ) as Json, +): SpanTreeEntry[] { + return buildRelevantEvents(events, operationSpecs).map( + ({ event, summaryName }) => { + const summary = summarizeOpenAIPayload(event, summaryName) as Record< + string, + Json + >; + const { name: _name, type: _type, ...fields } = summary; + + return { + event, + fields: { + span_attributes: spanTreeFields(event).span_attributes, + ...fields, + }, + name: + typeof summary.name === "string" + ? summary.name + : (summaryName ?? event.span.name), + }; + }, ); } export function defineOpenAIInstrumentationAssertions(options: { assertPrivateFieldMethodsOperation?: boolean; + cassetteName?: string; name: string; runScenario: RunOpenAIScenario; snapshotName: string; @@ -567,11 +557,7 @@ export function defineOpenAIInstrumentationAssertions(options: { const operationSpecs = getOperationSpecs(options.version); const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, - ); - const payloadSnapshotPath = resolveFileSnapshotPath( - options.testFileUrl, - `${options.snapshotName}.log-payloads.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -622,7 +608,7 @@ export function defineOpenAIInstrumentationAssertions(options: { path.join( scenarioDir, "__cassettes__", - `${options.snapshotName}.cassette.json`, + `${options.cassetteName ?? options.snapshotName}.cassette.json`, ), ); @@ -669,18 +655,8 @@ export function defineOpenAIInstrumentationAssertions(options: { }); } - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildSpanSummary(events, operationSpecs)), - spanSnapshotPath, - ); - }); - - test("matches the shared payload snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot(buildPayloadSummary(events, operationSpecs)), - payloadSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/openai-instrumentation/scenario.test.ts b/e2e/scenarios/openai-instrumentation/scenario.test.ts index 52106ef41..ecef247c3 100644 --- a/e2e/scenarios/openai-instrumentation/scenario.test.ts +++ b/e2e/scenarios/openai-instrumentation/scenario.test.ts @@ -80,7 +80,8 @@ describe.concurrent("variants", () => { timeoutMs: TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-wrapped`, + cassetteName: scenario.snapshotName, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, version: scenario.version, @@ -100,7 +101,8 @@ describe.concurrent("variants", () => { timeoutMs: TIMEOUT_MS, }); }, - snapshotName: scenario.snapshotName, + snapshotName: `${scenario.snapshotName}-auto-hook`, + cassetteName: scenario.snapshotName, testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, version: scenario.version, diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-events.json b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-events.json deleted file mode 100644 index d0f8ba9f5..000000000 --- a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-events.json +++ /dev/null @@ -1,294 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "openrouter-instrumentation" - }, - "metric_keys": [], - "name": "openrouter-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "openrouter-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_audio_tokens", - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_completions_cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_prompt_cost", - "prompt_audio_tokens", - "prompt_cache_write_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "prompt_video_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.chat.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "openrouter-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_audio_tokens", - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_completions_cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_prompt_cost", - "prompt_audio_tokens", - "prompt_cache_write_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "prompt_video_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.chat.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings" - }, - "metric_keys": [], - "name": "openrouter-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "embedding_model": "text-embedding-3-small", - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metric_keys": [ - "cost", - "prompt_tokens", - "tokens" - ], - "name": "openrouter.embeddings.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "rerank" - }, - "metric_keys": [], - "name": "openrouter-rerank-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "document_count": 3, - "model": "rerank-v3.5", - "provider": "cohere", - "topN": 2 - }, - "metric_keys": [ - "cost", - "search_units" - ], - "name": "openrouter.rerank.rerank", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses" - }, - "metric_keys": [], - "name": "openrouter-responses-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_input_cost", - "cost_upstream_inference_output_cost", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.beta.responses.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream" - }, - "metric_keys": [], - "name": "openrouter-responses-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_input_cost", - "cost_upstream_inference_output_cost", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.beta.responses.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "call-model" - }, - "metric_keys": [], - "name": "openrouter-call-model-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite", - "provider": "google" - }, - "metric_keys": [], - "name": "openrouter.callModel", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } -] diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.json b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.json new file mode 100644 index 000000000..92f1e71bf --- /dev/null +++ b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.json @@ -0,0 +1,1352 @@ +{ + "span_tree": [ + { + "name": "openrouter-root", + "type": "task", + "children": [ + { + "name": "openrouter-chat-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "OK.", + "reasoning": null, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "cost": 0.000003, + "cost_upstream_inference_completions_cost": 0.0000012, + "cost_upstream_inference_cost": 0.000003, + "cost_upstream_inference_prompt_cost": 0.0000018, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openrouter-chat-stream-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "streamOptions": { + "includeUsage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_tokens": 1, + "cost": 0.0000024, + "cost_upstream_inference_completions_cost": 6e-7, + "cost_upstream_inference_cost": 0.0000024, + "cost_upstream_inference_prompt_cost": 0.0000018, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-chat-reasoning-stream-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Think briefly, then answer with exactly the number 4.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "4", + "reasoning": "First, the user said: \"Think briefly, then answer with exactly the number 4.\" So, I need to think about this briefly before responding.\n\nThe instruction is straightforward: I should respond with the number 4, and nothing else. It says \"exactly the number 4,\" which probably means just the numeral \"4\" without any additional text.\n\nBut why would someone ask this? It might be a test to see if I can follow simple instructions precisely. Or perhaps it's part of a larger context, but in this message, it's isolated.\n\nLet me read it again: \"Think briefly, then answer with exactly the number 4.\" So, I should think briefly—which I am doing now—and then output \"4\".\n\nI need to ensure that my response is exactly \"4\". That means no extra spaces, no punctuation, just the digit 4.\n\nIs there any trick here? The user says \"think briefly,\" so I should acknowledge that internally. But in my response, I shouldn't include any of that thinking; I should only output the number.\n\nSo, after this brief thought process, I'll respond with \"4\".\n\nFinal decision: My response should be: 4\n", + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 256, + "model": "deepseek-r1", + "provider": "deepseek", + "reasoning": { + "enabled": true, + "exclude": false + }, + "stream": true, + "streamOptions": { + "includeUsage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 247, + "completion_tokens": 249, + "cost": 0.0006344, + "cost_upstream_inference_completions_cost": 0.0006225, + "cost_upstream_inference_cost": 0.0006344, + "cost_upstream_inference_prompt_cost": 0.0000119, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 266 + } + } + ], + "metadata": { + "operation": "chat-reasoning-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-embeddings-operation", + "children": [ + { + "name": "openrouter.embeddings.generate", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "embedding_model": "text-embedding-3-small", + "id": "", + "inputType": "query", + "model": "text-embedding-3-small", + "object": "list", + "provider": "openai" + }, + "metrics": { + "cost": 6e-8, + "prompt_tokens": 3, + "tokens": 3 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openrouter-responses-operation", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Reply with exactly OBSERVABILITY.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "OBSERVABILITY", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095505, + "createdAt": 1779095504, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "openai", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "auto", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "auto", + "tools": [], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "cost": 0.0000039, + "cost_upstream_inference_cost": 0.0000039, + "cost_upstream_inference_input_cost": 0.0000021, + "cost_upstream_inference_output_cost": 0.0000018, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openrouter-responses-stream-operation", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Reply with exactly STREAMED RESPONSE.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "STREAMED RESPONSE", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095506, + "createdAt": 1779095505, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "openai", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "auto", + "status": "completed", + "store": false, + "stream": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "auto", + "tools": [], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "cost": 0.0000039, + "cost_upstream_inference_cost": 0.0000039, + "cost_upstream_inference_input_cost": 0.0000021, + "cost_upstream_inference_output_cost": 0.0000018, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-rerank-operation", + "children": [ + { + "name": "openrouter.rerank.rerank", + "type": "llm", + "children": [], + "input": { + "documents": [ + "Athens is in Greece.", + "Paris is in France.", + "Lima is in Peru." + ], + "query": "Which document is about France?" + }, + "output": [ + { + "index": 1, + "relevance_score": 0.06664825 + }, + { + "index": 0, + "relevance_score": 0.015062517 + } + ], + "metadata": { + "document_count": 3, + "id": "", + "model": "rerank-v3.5", + "provider": "cohere", + "results": [ + { + "document": { + "text": "Paris is in France." + }, + "index": 1, + "relevanceScore": 0.06664825 + }, + { + "document": { + "text": "Athens is in Greece." + }, + "index": 0, + "relevanceScore": 0.015062517 + } + ], + "topN": 2 + }, + "metrics": { + "cost": 0.001, + "search_units": 1 + } + } + ], + "metadata": { + "operation": "rerank", + "testRunId": "" + } + }, + { + "name": "openrouter-call-model-operation", + "children": [ + { + "name": "openrouter.callModel", + "type": "task", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095506, + "createdAt": 1779095506, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 1, + "step_type": "initial", + "store": false, + "stream": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000054, + "cost_upstream_inference_cost": 0.0000054, + "cost_upstream_inference_input_cost": 0.0000034, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 34, + "tokens": 39 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095507, + "createdAt": 1779095506, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 2, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000083, + "cost_upstream_inference_cost": 0.0000083, + "cost_upstream_inference_input_cost": 0.0000063, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 63, + "tokens": 68 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095507, + "createdAt": 1779095507, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 3, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000111, + "cost_upstream_inference_cost": 0.0000111, + "cost_upstream_inference_input_cost": 0.0000091, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 91, + "tokens": 96 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095507, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 4, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000142, + "cost_upstream_inference_cost": 0.0000142, + "cost_upstream_inference_input_cost": 0.0000122, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 122, + "tokens": 127 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095508, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 5, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000173, + "cost_upstream_inference_cost": 0.0000173, + "cost_upstream_inference_input_cost": 0.0000153, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 153, + "tokens": 158 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_IlQOLzucJR2JyCc4ZvYC", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095508, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 6, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000199, + "cost_upstream_inference_cost": 0.0000199, + "cost_upstream_inference_input_cost": 0.0000179, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 179, + "tokens": 184 + } + } + ], + "input": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_IlQOLzucJR2JyCc4ZvYC", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095508, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled", + "turn_count": 6 + } + } + ], + "metadata": { + "operation": "call-model", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "openrouter-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.txt b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.txt new file mode 100644 index 000000000..96ca6ce42 --- /dev/null +++ b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0123.span-tree.txt @@ -0,0 +1,1235 @@ +span_tree: +└── openrouter-root [task] + metadata: { + "scenario": "openrouter-instrumentation", + "testRunId": "" + } + ├── openrouter-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "OK.", + │ "reasoning": null, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "cost": 0.000003, + │ "cost_upstream_inference_completions_cost": 0.0000012, + │ "cost_upstream_inference_cost": 0.000003, + │ "cost_upstream_inference_prompt_cost": 0.0000018, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openrouter-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "streamOptions": { + │ "includeUsage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 1, + │ "cost": 0.0000024, + │ "cost_upstream_inference_completions_cost": 6e-7, + │ "cost_upstream_inference_cost": 0.0000024, + │ "cost_upstream_inference_prompt_cost": 0.0000018, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openrouter-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-reasoning-stream", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Think briefly, then answer with exactly the number 4.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "4", + │ "reasoning": "First, the user said: \"Think briefly, then answer with exactly the number 4.\" So, I need to think about this briefly before responding.\n\nThe instruction is straightforward: I should respond with the number 4, and nothing else. It says \"exactly the number 4,\" which probably means just the numeral \"4\" without any additional text.\n\nBut why would someone ask this? It might be a test to see if I can follow simple instructions precisely. Or perhaps it's part of a larger context, but in this message, it's isolated.\n\nLet me read it again: \"Think briefly, then answer with exactly the number 4.\" So, I should think briefly—which I am doing now—and then output \"4\".\n\nI need to ensure that my response is exactly \"4\". That means no extra spaces, no punctuation, just the digit 4.\n\nIs there any trick here? The user says \"think briefly,\" so I should acknowledge that internally. But in my response, I shouldn't include any of that thinking; I should only output the number.\n\nSo, after this brief thought process, I'll respond with \"4\".\n\nFinal decision: My response should be: 4\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 256, + │ "model": "deepseek-r1", + │ "provider": "deepseek", + │ "reasoning": { + │ "enabled": true, + │ "exclude": false + │ }, + │ "stream": true, + │ "streamOptions": { + │ "includeUsage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 247, + │ "completion_tokens": 249, + │ "cost": 0.0006344, + │ "cost_upstream_inference_completions_cost": 0.0006225, + │ "cost_upstream_inference_cost": 0.0006344, + │ "cost_upstream_inference_prompt_cost": 0.0000119, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 266 + │ } + ├── openrouter-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── openrouter.embeddings.generate [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "embedding_model": "text-embedding-3-small", + │ "id": "", + │ "inputType": "query", + │ "model": "text-embedding-3-small", + │ "object": "list", + │ "provider": "openai" + │ } + │ metrics: { + │ "cost": 6e-8, + │ "prompt_tokens": 3, + │ "tokens": 3 + │ } + ├── openrouter-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openrouter.beta.responses.send [llm] + │ input: "Reply with exactly OBSERVABILITY." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "OBSERVABILITY", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095505, + │ "createdAt": 1779095504, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "openai", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "auto", + │ "status": "completed", + │ "store": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "auto", + │ "tools": [], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "cost": 0.0000039, + │ "cost_upstream_inference_cost": 0.0000039, + │ "cost_upstream_inference_input_cost": 0.0000021, + │ "cost_upstream_inference_output_cost": 0.0000018, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openrouter-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openrouter.beta.responses.send [llm] + │ input: "Reply with exactly STREAMED RESPONSE." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "STREAMED RESPONSE", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095506, + │ "createdAt": 1779095505, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "openai", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "auto", + │ "status": "completed", + │ "store": false, + │ "stream": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "auto", + │ "tools": [], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "cost": 0.0000039, + │ "cost_upstream_inference_cost": 0.0000039, + │ "cost_upstream_inference_input_cost": 0.0000021, + │ "cost_upstream_inference_output_cost": 0.0000018, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openrouter-rerank-operation + │ metadata: { + │ "operation": "rerank", + │ "testRunId": "" + │ } + │ └── openrouter.rerank.rerank [llm] + │ input: { + │ "documents": [ + │ "Athens is in Greece.", + │ "Paris is in France.", + │ "Lima is in Peru." + │ ], + │ "query": "Which document is about France?" + │ } + │ output: [ + │ { + │ "index": 1, + │ "relevance_score": 0.06664825 + │ }, + │ { + │ "index": 0, + │ "relevance_score": 0.015062517 + │ } + │ ] + │ metadata: { + │ "document_count": 3, + │ "id": "", + │ "model": "rerank-v3.5", + │ "provider": "cohere", + │ "results": [ + │ { + │ "document": { + │ "text": "Paris is in France." + │ }, + │ "index": 1, + │ "relevanceScore": 0.06664825 + │ }, + │ { + │ "document": { + │ "text": "Athens is in Greece." + │ }, + │ "index": 0, + │ "relevanceScore": 0.015062517 + │ } + │ ], + │ "topN": 2 + │ } + │ metrics: { + │ "cost": 0.001, + │ "search_units": 1 + │ } + └── openrouter-call-model-operation + metadata: { + "operation": "call-model", + "testRunId": "" + } + └── openrouter.callModel [task] + input: "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast." + output: [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_IlQOLzucJR2JyCc4ZvYC", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ] + metadata: { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095508, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled", + "turn_count": 6 + } + ├── openrouter.beta.responses.send [llm] + │ input: "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast." + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095506, + │ "createdAt": 1779095506, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 1, + │ "step_type": "initial", + │ "store": false, + │ "stream": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000054, + │ "cost_upstream_inference_cost": 0.0000054, + │ "cost_upstream_inference_input_cost": 0.0000034, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 34, + │ "tokens": 39 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095507, + │ "createdAt": 1779095506, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 2, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000083, + │ "cost_upstream_inference_cost": 0.0000083, + │ "cost_upstream_inference_input_cost": 0.0000063, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 63, + │ "tokens": 68 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095507, + │ "createdAt": 1779095507, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 3, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000111, + │ "cost_upstream_inference_cost": 0.0000111, + │ "cost_upstream_inference_input_cost": 0.0000091, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 91, + │ "tokens": 96 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095508, + │ "createdAt": 1779095507, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 4, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000142, + │ "cost_upstream_inference_cost": 0.0000142, + │ "cost_upstream_inference_input_cost": 0.0000122, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 122, + │ "tokens": 127 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095508, + │ "createdAt": 1779095508, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 5, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000173, + │ "cost_upstream_inference_cost": 0.0000173, + │ "cost_upstream_inference_input_cost": 0.0000153, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 153, + │ "tokens": 158 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + └── openrouter.beta.responses.send [llm] + input: [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_W2OBzHgZJzYCuUWlIRKD", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_32fBhDlHpSv7ZH1GlaSZ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_d9jmcL6mO40uuOCeEGW9", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_rMgILn38GM5g1U6z6mmJ", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_kufeiiPIWmxUMoDEFcEr", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ] + output: [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_IlQOLzucJR2JyCc4ZvYC", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ] + metadata: { + "background": false, + "completedAt": 1779095508, + "createdAt": 1779095508, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 6, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000199, + "cost_upstream_inference_cost": 0.0000199, + "cost_upstream_inference_input_cost": 0.0000179, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 179, + "tokens": 184 + } diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-events.json b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-events.json deleted file mode 100644 index a0a45683f..000000000 --- a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-events.json +++ /dev/null @@ -1,250 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "openrouter-instrumentation" - }, - "metric_keys": [], - "name": "openrouter-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat" - }, - "metric_keys": [], - "name": "openrouter-chat-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_audio_tokens", - "completion_reasoning_tokens", - "completion_tokens", - "prompt_audio_tokens", - "prompt_cache_write_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "prompt_video_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.chat.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chat-stream" - }, - "metric_keys": [], - "name": "openrouter-chat-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_audio_tokens", - "completion_reasoning_tokens", - "completion_tokens", - "prompt_audio_tokens", - "prompt_cache_write_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "prompt_video_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.chat.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "embeddings" - }, - "metric_keys": [], - "name": "openrouter-embeddings-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "embedding_model": "text-embedding-3-small", - "model": "text-embedding-3-small", - "provider": "openai" - }, - "metric_keys": [ - "cost", - "prompt_tokens", - "tokens" - ], - "name": "openrouter.embeddings.generate", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses" - }, - "metric_keys": [], - "name": "openrouter-responses-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_input_cost", - "cost_upstream_inference_output_cost", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.beta.responses.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "responses-stream" - }, - "metric_keys": [], - "name": "openrouter-responses-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18", - "provider": "openai" - }, - "metric_keys": [ - "completion_reasoning_tokens", - "completion_tokens", - "cost", - "cost_upstream_inference_cost", - "cost_upstream_inference_input_cost", - "cost_upstream_inference_output_cost", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "tokens" - ], - "name": "openrouter.beta.responses.send", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "call-model" - }, - "metric_keys": [], - "name": "openrouter-call-model-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gemini-2.5-flash-lite", - "provider": "google" - }, - "metric_keys": [], - "name": "openrouter.callModel", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - } -] diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.json b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.json new file mode 100644 index 000000000..6403673ae --- /dev/null +++ b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.json @@ -0,0 +1,1281 @@ +{ + "span_tree": [ + { + "name": "openrouter-root", + "type": "task", + "children": [ + { + "name": "openrouter-chat-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finishReason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "OK.", + "reasoning": null, + "refusal": null, + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_tokens": 2, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 14 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "openrouter-chat-stream-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "STREAM", + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "provider": "openai", + "stream": true, + "streamOptions": { + "includeUsage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 0, + "completion_tokens": 1, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 13 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-chat-reasoning-stream-operation", + "children": [ + { + "name": "openrouter.chat.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Think briefly, then answer with exactly the number 4.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "4", + "reasoning": "First, the user said: \"Think briefly, then answer with exactly the number 4.\" So, I need to think briefly about this, but then my response should be just the number 4, nothing else.\n\nThe instruction is clear: after thinking, I must respond with \"4\". No additional text, no explanations, just the numeral 4.\n\nNow, why would the user ask this? It might be a test to see if I can follow simple instructions precisely. Or it could be part of a larger context, but in this message, it's standalone.\n\nI should think briefly as instructed. What is there to think about? The request is straightforward. I could consider why they want the number 4, but that might be overcomplicating it. The key is to comply with the instruction.\n\nThinking briefly: The user wants me to output the number 4. That's it. No need for any complex reasoning.\n\nTherefore, my response should be: 4\n\nExactly that. Not \"four\" in words, not with a period, just \"4\".\n\nI recall that in programming or AI contexts, sometimes users test if the model can output exactly what's asked without embellishment.\n\nSo, I'll do exactly as instructed.\n", + "role": "assistant" + } + } + ], + "metadata": { + "maxTokens": 256, + "model": "deepseek-r1", + "provider": "deepseek", + "reasoning": { + "enabled": true, + "exclude": false + }, + "stream": true, + "streamOptions": { + "includeUsage": true + }, + "temperature": 0 + }, + "metrics": { + "completion_audio_tokens": 0, + "completion_reasoning_tokens": 253, + "completion_tokens": 255, + "prompt_audio_tokens": 0, + "prompt_cache_write_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 17, + "prompt_video_tokens": 0, + "time_to_first_token": 0, + "tokens": 272 + } + } + ], + "metadata": { + "operation": "chat-reasoning-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-embeddings-operation", + "children": [ + { + "name": "openrouter.embeddings.generate", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 1536 + }, + "metadata": { + "embedding_model": "text-embedding-3-small", + "id": "", + "inputType": "query", + "model": "text-embedding-3-small", + "object": "list", + "provider": "openai" + }, + "metrics": { + "cost": 6e-8, + "prompt_tokens": 3, + "tokens": 3 + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "openrouter-responses-operation", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Reply with exactly OBSERVABILITY.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "OBSERVABILITY", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095456, + "createdAt": 1779095455, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "openai", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "auto", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "auto", + "tools": [], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "cost": 0.0000039, + "cost_upstream_inference_cost": 0.0000039, + "cost_upstream_inference_input_cost": 0.0000021, + "cost_upstream_inference_output_cost": 0.0000018, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses", + "testRunId": "" + } + }, + { + "name": "openrouter-responses-stream-operation", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Reply with exactly STREAMED RESPONSE.", + "output": [ + { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "STREAMED RESPONSE", + "type": "output_text" + } + ], + "id": "", + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095457, + "createdAt": 1779095456, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": null, + "metadata": {}, + "model": "gpt-4o-mini-2024-07-18", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "openai", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "auto", + "status": "completed", + "store": false, + "stream": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "auto", + "tools": [], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 3, + "cost": 0.0000039, + "cost_upstream_inference_cost": 0.0000039, + "cost_upstream_inference_input_cost": 0.0000021, + "cost_upstream_inference_output_cost": 0.0000018, + "prompt_cached_tokens": 0, + "prompt_tokens": 14, + "time_to_first_token": 0, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "responses-stream", + "testRunId": "" + } + }, + { + "name": "openrouter-call-model-operation", + "children": [ + { + "name": "openrouter.callModel", + "type": "task", + "children": [ + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095457, + "createdAt": 1779095457, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": null, + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 1, + "step_type": "initial", + "store": false, + "stream": true, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000054, + "cost_upstream_inference_cost": 0.0000054, + "cost_upstream_inference_input_cost": 0.0000034, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 34, + "tokens": 39 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095458, + "createdAt": 1779095457, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 2, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000081, + "cost_upstream_inference_cost": 0.0000081, + "cost_upstream_inference_input_cost": 0.0000061, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 61, + "tokens": 66 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095458, + "createdAt": 1779095458, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 3, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000111, + "cost_upstream_inference_cost": 0.0000111, + "cost_upstream_inference_input_cost": 0.0000091, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 91, + "tokens": 96 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095458, + "createdAt": 1779095458, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 4, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.000014, + "cost_upstream_inference_cost": 0.000014, + "cost_upstream_inference_input_cost": 0.000012, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 120, + "tokens": 125 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095459, + "createdAt": 1779095459, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 5, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.0000171, + "cost_upstream_inference_cost": 0.0000171, + "cost_upstream_inference_input_cost": 0.0000151, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 151, + "tokens": 156 + } + }, + { + "name": "lookup_weather", + "type": "tool", + "children": [], + "input": { + "city": "Vienna" + }, + "output": { + "forecast": "Sunny in Vienna" + }, + "metadata": { + "provider": "openrouter", + "tool_name": "lookup_weather" + } + }, + { + "name": "openrouter.beta.responses.send", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ], + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_15xoKamTLYmR70wyXZe3", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095459, + "createdAt": 1779095459, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 6, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + }, + "metrics": { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.00002, + "cost_upstream_inference_cost": 0.00002, + "cost_upstream_inference_input_cost": 0.000018, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 180, + "tokens": 185 + } + } + ], + "input": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "output": [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_15xoKamTLYmR70wyXZe3", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ], + "metadata": { + "background": false, + "completedAt": 1779095459, + "createdAt": 1779095459, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled", + "turn_count": 6 + } + } + ], + "metadata": { + "operation": "call-model", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "openrouter-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.txt b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.txt new file mode 100644 index 000000000..54e3a2673 --- /dev/null +++ b/e2e/scenarios/openrouter-instrumentation/__snapshots__/openrouter-v0911.span-tree.txt @@ -0,0 +1,1172 @@ +span_tree: +└── openrouter-root [task] + metadata: { + "scenario": "openrouter-instrumentation", + "testRunId": "" + } + ├── openrouter-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finishReason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "OK.", + │ "reasoning": null, + │ "refusal": null, + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 2, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 14 + │ } + ├── openrouter-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "STREAM", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "provider": "openai", + │ "stream": true, + │ "streamOptions": { + │ "includeUsage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 1, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 13 + │ } + ├── openrouter-chat-reasoning-stream-operation + │ metadata: { + │ "operation": "chat-reasoning-stream", + │ "testRunId": "" + │ } + │ └── openrouter.chat.send [llm] + │ input: [ + │ { + │ "content": "Think briefly, then answer with exactly the number 4.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "logprobs": null, + │ "message": { + │ "content": "4", + │ "reasoning": "First, the user said: \"Think briefly, then answer with exactly the number 4.\" So, I need to think briefly about this, but then my response should be just the number 4, nothing else.\n\nThe instruction is clear: after thinking, I must respond with \"4\". No additional text, no explanations, just the numeral 4.\n\nNow, why would the user ask this? It might be a test to see if I can follow simple instructions precisely. Or it could be part of a larger context, but in this message, it's standalone.\n\nI should think briefly as instructed. What is there to think about? The request is straightforward. I could consider why they want the number 4, but that might be overcomplicating it. The key is to comply with the instruction.\n\nThinking briefly: The user wants me to output the number 4. That's it. No need for any complex reasoning.\n\nTherefore, my response should be: 4\n\nExactly that. Not \"four\" in words, not with a period, just \"4\".\n\nI recall that in programming or AI contexts, sometimes users test if the model can output exactly what's asked without embellishment.\n\nSo, I'll do exactly as instructed.\n", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "maxTokens": 256, + │ "model": "deepseek-r1", + │ "provider": "deepseek", + │ "reasoning": { + │ "enabled": true, + │ "exclude": false + │ }, + │ "stream": true, + │ "streamOptions": { + │ "includeUsage": true + │ }, + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_audio_tokens": 0, + │ "completion_reasoning_tokens": 253, + │ "completion_tokens": 255, + │ "prompt_audio_tokens": 0, + │ "prompt_cache_write_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 17, + │ "prompt_video_tokens": 0, + │ "time_to_first_token": 0, + │ "tokens": 272 + │ } + ├── openrouter-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── openrouter.embeddings.generate [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 1536 + │ } + │ metadata: { + │ "embedding_model": "text-embedding-3-small", + │ "id": "", + │ "inputType": "query", + │ "model": "text-embedding-3-small", + │ "object": "list", + │ "provider": "openai" + │ } + │ metrics: { + │ "cost": 6e-8, + │ "prompt_tokens": 3, + │ "tokens": 3 + │ } + ├── openrouter-responses-operation + │ metadata: { + │ "operation": "responses", + │ "testRunId": "" + │ } + │ └── openrouter.beta.responses.send [llm] + │ input: "Reply with exactly OBSERVABILITY." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "OBSERVABILITY", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095456, + │ "createdAt": 1779095455, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "openai", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "auto", + │ "status": "completed", + │ "store": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "auto", + │ "tools": [], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "cost": 0.0000039, + │ "cost_upstream_inference_cost": 0.0000039, + │ "cost_upstream_inference_input_cost": 0.0000021, + │ "cost_upstream_inference_output_cost": 0.0000018, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + ├── openrouter-responses-stream-operation + │ metadata: { + │ "operation": "responses-stream", + │ "testRunId": "" + │ } + │ └── openrouter.beta.responses.send [llm] + │ input: "Reply with exactly STREAMED RESPONSE." + │ output: [ + │ { + │ "content": [ + │ { + │ "annotations": [], + │ "logprobs": [], + │ "text": "STREAMED RESPONSE", + │ "type": "output_text" + │ } + │ ], + │ "id": "", + │ "role": "assistant", + │ "status": "completed", + │ "type": "message" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095457, + │ "createdAt": 1779095456, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": null, + │ "metadata": {}, + │ "model": "gpt-4o-mini-2024-07-18", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "openai", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "auto", + │ "status": "completed", + │ "store": false, + │ "stream": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "auto", + │ "tools": [], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 3, + │ "cost": 0.0000039, + │ "cost_upstream_inference_cost": 0.0000039, + │ "cost_upstream_inference_input_cost": 0.0000021, + │ "cost_upstream_inference_output_cost": 0.0000018, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 14, + │ "time_to_first_token": 0, + │ "tokens": 17 + │ } + └── openrouter-call-model-operation + metadata: { + "operation": "call-model", + "testRunId": "" + } + └── openrouter.callModel [task] + input: "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast." + output: [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_15xoKamTLYmR70wyXZe3", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ] + metadata: { + "background": false, + "completedAt": 1779095459, + "createdAt": 1779095459, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "store": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled", + "turn_count": 6 + } + ├── openrouter.beta.responses.send [llm] + │ input: "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast." + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095457, + │ "createdAt": 1779095457, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": null, + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 1, + │ "step_type": "initial", + │ "store": false, + │ "stream": true, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000054, + │ "cost_upstream_inference_cost": 0.0000054, + │ "cost_upstream_inference_input_cost": 0.0000034, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 34, + │ "tokens": 39 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095458, + │ "createdAt": 1779095457, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 2, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000081, + │ "cost_upstream_inference_cost": 0.0000081, + │ "cost_upstream_inference_input_cost": 0.0000061, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 61, + │ "tokens": 66 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095458, + │ "createdAt": 1779095458, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 3, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000111, + │ "cost_upstream_inference_cost": 0.0000111, + │ "cost_upstream_inference_input_cost": 0.0000091, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 91, + │ "tokens": 96 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095458, + │ "createdAt": 1779095458, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 4, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.000014, + │ "cost_upstream_inference_cost": 0.000014, + │ "cost_upstream_inference_input_cost": 0.000012, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 120, + │ "tokens": 125 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + ├── openrouter.beta.responses.send [llm] + │ input: [ + │ { + │ "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + │ "role": "user" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ }, + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ }, + │ { + │ "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + │ "id": "", + │ "output": "{\"forecast\":\"Sunny in Vienna\"}", + │ "type": "function_call_output" + │ } + │ ] + │ output: [ + │ { + │ "arguments": "{\"city\":\"Vienna\"}", + │ "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + │ "id": "", + │ "name": "lookup_weather", + │ "status": "completed", + │ "type": "function_call" + │ } + │ ] + │ metadata: { + │ "background": false, + │ "completedAt": 1779095459, + │ "createdAt": 1779095459, + │ "error": null, + │ "frequencyPenalty": 0, + │ "id": "", + │ "incompleteDetails": null, + │ "instructions": null, + │ "is_byok": false, + │ "maxOutputTokens": 24, + │ "maxToolCalls": 1, + │ "metadata": {}, + │ "model": "gemini-2.5-flash-lite", + │ "object": "response", + │ "parallelToolCalls": true, + │ "presencePenalty": 0, + │ "previousResponseId": null, + │ "promptCacheKey": "[REDACTED]", + │ "provider": "google", + │ "reasoning": null, + │ "safetyIdentifier": null, + │ "serviceTier": "standard", + │ "status": "completed", + │ "step": 5, + │ "step_type": "continue", + │ "store": false, + │ "stream": false, + │ "temperature": 0, + │ "text": { + │ "format": { + │ "type": "text" + │ } + │ }, + │ "toolChoice": "required", + │ "tools": [ + │ { + │ "description": "Look up the weather forecast for a city.", + │ "name": "lookup_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ }, + │ "strict": null, + │ "type": "function" + │ } + │ ], + │ "topLogprobs": 0, + │ "topP": 1, + │ "truncation": "disabled" + │ } + │ metrics: { + │ "completion_reasoning_tokens": 0, + │ "completion_tokens": 5, + │ "cost": 0.0000171, + │ "cost_upstream_inference_cost": 0.0000171, + │ "cost_upstream_inference_input_cost": 0.0000151, + │ "cost_upstream_inference_output_cost": 0.000002, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 151, + │ "tokens": 156 + │ } + ├── lookup_weather [tool] + │ input: { + │ "city": "Vienna" + │ } + │ output: { + │ "forecast": "Sunny in Vienna" + │ } + │ metadata: { + │ "provider": "openrouter", + │ "tool_name": "lookup_weather" + │ } + └── openrouter.beta.responses.send [llm] + input: [ + { + "content": "Use the lookup_weather tool for Vienna exactly once, then answer with only the forecast.", + "role": "user" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_ymMgyTftjllIoMk4ENiy", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_psskNCvZbY12v83qWva4", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_vvaJyggWXRmK9zQti8Ql", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_e24THvE7S30bxnPvtvh7", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + }, + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + }, + { + "callId": "tool_lookup_weather_G3tzdNelHAmKXHiU0i4k", + "id": "", + "output": "{\"forecast\":\"Sunny in Vienna\"}", + "type": "function_call_output" + } + ] + output: [ + { + "arguments": "{\"city\":\"Vienna\"}", + "callId": "tool_lookup_weather_15xoKamTLYmR70wyXZe3", + "id": "", + "name": "lookup_weather", + "status": "completed", + "type": "function_call" + } + ] + metadata: { + "background": false, + "completedAt": 1779095459, + "createdAt": 1779095459, + "error": null, + "frequencyPenalty": 0, + "id": "", + "incompleteDetails": null, + "instructions": null, + "is_byok": false, + "maxOutputTokens": 24, + "maxToolCalls": 1, + "metadata": {}, + "model": "gemini-2.5-flash-lite", + "object": "response", + "parallelToolCalls": true, + "presencePenalty": 0, + "previousResponseId": null, + "promptCacheKey": "[REDACTED]", + "provider": "google", + "reasoning": null, + "safetyIdentifier": null, + "serviceTier": "standard", + "status": "completed", + "step": 6, + "step_type": "continue", + "store": false, + "stream": false, + "temperature": 0, + "text": { + "format": { + "type": "text" + } + }, + "toolChoice": "required", + "tools": [ + { + "description": "Look up the weather forecast for a city.", + "name": "lookup_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "strict": null, + "type": "function" + } + ], + "topLogprobs": 0, + "topP": 1, + "truncation": "disabled" + } + metrics: { + "completion_reasoning_tokens": 0, + "completion_tokens": 5, + "cost": 0.00002, + "cost_upstream_inference_cost": 0.00002, + "cost_upstream_inference_input_cost": 0.000018, + "cost_upstream_inference_output_cost": 0.000002, + "prompt_cached_tokens": 0, + "prompt_tokens": 180, + "tokens": 185 + } diff --git a/e2e/scenarios/openrouter-instrumentation/assertions.ts b/e2e/scenarios/openrouter-instrumentation/assertions.ts index 68542abd1..bbbc44619 100644 --- a/e2e/scenarios/openrouter-instrumentation/assertions.ts +++ b/e2e/scenarios/openrouter-instrumentation/assertions.ts @@ -1,17 +1,12 @@ import { beforeAll, describe, expect, test } from "vitest"; -import type { Json } from "../../helpers/normalize"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { withScenarioHarness, type ScenarioRunContext, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; import { CALL_MODEL, @@ -78,7 +73,7 @@ function findOpenRouterSpan( function buildSpanSummary( events: CapturedLogEvent[], options: { supportsRerank: boolean }, -): Json { +): CapturedLogEvent[] { const chatOperation = findLatestSpan(events, "openrouter-chat-operation"); const chatStreamOperation = findLatestSpan( events, @@ -136,17 +131,7 @@ function buildSpanSummary( findOpenRouterSpan(events, callModelOperation?.span.id, [ "openrouter.callModel", ]), - ].map((event) => - summarizeWrapperContract(event!, [ - "document_count", - "embedding_model", - "model", - "operation", - "provider", - "scenario", - "topN", - ]), - ) as Json; + ].map((event) => event!); } export function defineOpenRouterTraceAssertions(options: { @@ -159,7 +144,7 @@ export function defineOpenRouterTraceAssertions(options: { }): void { const spanSnapshotPath = resolveFileSnapshotPath( options.testFileUrl, - `${options.snapshotName}.span-events.json`, + `${options.snapshotName}.span-tree.json`, ); const testConfig = { timeout: options.timeoutMs, @@ -409,15 +394,8 @@ export function defineOpenRouterTraceAssertions(options: { }, ); - test("matches the shared span snapshot", testConfig, async () => { - await matchFileSnapshot( - formatJsonFileSnapshot( - buildSpanSummary(events, { - supportsRerank: options.supportsRerank, - }), - ), - spanSnapshotPath, - ); + test("matches the shared span tree snapshot", testConfig, async () => { + await matchSpanTreeSnapshot(events, spanSnapshotPath); }); }); } diff --git a/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.json b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.json new file mode 100644 index 000000000..f85469bcd --- /dev/null +++ b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.json @@ -0,0 +1,25 @@ +{ + "span_tree": [ + { + "name": "bt-root", + "type": "task", + "children": [], + "metadata": { + "scenario": "otel-compat-mixed-tracing", + "testRunId": "" + } + }, + { + "name": "bt-child-under-otel", + "type": "task", + "children": [], + "output": { + "source": "otel-child-context" + }, + "metadata": { + "kind": "bt-child-under-otel", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.txt b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.txt new file mode 100644 index 000000000..14652c0a8 --- /dev/null +++ b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/braintrust-span-tree.txt @@ -0,0 +1,14 @@ +span_tree: +├── bt-root [task] +│ metadata: { +│ "scenario": "otel-compat-mixed-tracing", +│ "testRunId": "" +│ } +└── bt-child-under-otel [task] + output: { + "source": "otel-child-context" + } + metadata: { + "kind": "bt-child-under-otel", + "testRunId": "" + } diff --git a/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap deleted file mode 100644 index 8ee531240..000000000 --- a/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap +++ /dev/null @@ -1,46 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`otel-compat-mixed-tracing unifies Braintrust and OTEL spans into one trace > braintrust-span-events 1`] = ` -[ - { - "error": null, - "input": null, - "metadata": { - "scenario": "otel-compat-mixed-tracing", - "testRunId": "", - }, - "name": "bt-root", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "bt-root", - "type": "task", - }, - "span_id": "", - "span_parents": null, - }, - { - "error": null, - "input": null, - "metadata": { - "kind": "bt-child-under-otel", - "testRunId": "", - }, - "name": "bt-child-under-otel", - "output": { - "source": "otel-child-context", - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "bt-child-under-otel", - "type": "task", - }, - "span_id": "", - "span_parents": [ - "", - ], - }, -] -`; diff --git a/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts b/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts index f6f9f5335..bd91dc47b 100644 --- a/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts +++ b/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts @@ -1,16 +1,21 @@ import { expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { extractOtelSpans, summarizeEvent } from "../../helpers/trace-summary"; +import { extractOtelSpans } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); +const spanTreeSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "braintrust-span-tree.json", +); test("otel-compat-mixed-tracing unifies Braintrust and OTEL spans into one trace", async () => { await withScenarioHarness( @@ -39,11 +44,7 @@ test("otel-compat-mixed-tracing unifies Braintrust and OTEL spans into one trace expect(btChild?.span.rootId).toBe(btRoot?.span.rootId); expect(btChild?.span.parentIds).toContain(otelMiddle?.spanId ?? ""); - expect( - normalizeForSnapshot( - [btRoot, btChild].map((event) => summarizeEvent(event!)) as Json, - ), - ).toMatchSnapshot("braintrust-span-events"); + await matchSpanTreeSnapshot(btEvents, spanTreeSnapshotPath); }, ); }); diff --git a/e2e/scenarios/test-framework-evals-jest/__snapshots__/log-payloads.json b/e2e/scenarios/test-framework-evals-jest/__snapshots__/log-payloads.json deleted file mode 100644 index 303f72b38..000000000 --- a/e2e/scenarios/test-framework-evals-jest/__snapshots__/log-payloads.json +++ /dev/null @@ -1,262 +0,0 @@ -[ - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "Object.", - "caller_lineno": 0 - }, - "created": "", - "expected": "Paris", - "id": "", - "input": "What is the capital of France?", - "log_id": "g", - "metadata": { - "case": "basic-span", - "scenario": "test-framework-evals-jest", - "testRunId": "", - "transport": "http" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": "Paris", - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "jest basic span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "Object.", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "log_id": "g", - "metadata": { - "case": "json-attachment", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "attachment": true - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "root", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "Object.", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "phase": "parent", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "parent-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "parent" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "jest parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "step": "child", - "testRunId": "" - }, - "log_id": "g", - "metadata": { - "case": "child-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "ok": true, - "phase": "child" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "jest child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "Object.", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-parent", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "jest nested parent span", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-child", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "jest nested child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "traced.name", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "nested-grandchild", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "depth": 3 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "jest nested grandchild span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/scenarios/test-framework-evals-jest/runner.case.cjs", - "caller_functionname": "Object.", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "case": "current-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "observedSpanId": "" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "jest current span", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-events.json b/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-events.json deleted file mode 100644 index e92c220e6..000000000 --- a/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-events.json +++ /dev/null @@ -1,184 +0,0 @@ -[ - { - "error": null, - "input": "What is the capital of France?", - "metadata": { - "case": "basic-span", - "scenario": "test-framework-evals-jest", - "testRunId": "", - "transport": "http" - }, - "name": "jest basic span", - "output": "Paris", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "jest basic span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "transcript": { - "content_type": "application/json", - "filename": "conversation_transcript.json", - "key": "", - "type": "braintrust_attachment" - }, - "type": "chat_completion" - }, - "metadata": { - "case": "json-attachment", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "root", - "output": { - "attachment": true - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "root", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "phase": "parent", - "testRunId": "" - }, - "metadata": { - "case": "parent-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest parent span", - "output": { - "ok": true, - "phase": "parent" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "jest parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": { - "step": "child", - "testRunId": "" - }, - "metadata": { - "case": "child-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest child span", - "output": { - "ok": true, - "phase": "child" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "jest child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-parent", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest nested parent span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "jest nested parent span", - "type": "task" - }, - "span_id": "", - "span_parents": null - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-child", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest nested child span", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "jest nested child span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "nested-grandchild", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest nested grandchild span", - "output": { - "depth": 3 - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "jest nested grandchild span" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "error": null, - "input": null, - "metadata": { - "case": "current-span", - "scenario": "test-framework-evals-jest", - "testRunId": "" - }, - "name": "jest current span", - "output": { - "observedSpanId": "" - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "jest current span", - "type": "task" - }, - "span_id": "", - "span_parents": null - } -] diff --git a/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.json b/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.json new file mode 100644 index 000000000..6248a909f --- /dev/null +++ b/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.json @@ -0,0 +1,122 @@ +{ + "span_tree": [ + { + "name": "jest basic span", + "type": "task", + "children": [], + "input": "What is the capital of France?", + "output": "Paris", + "expected": "Paris", + "metadata": { + "case": "basic-span", + "scenario": "test-framework-evals-jest", + "testRunId": "", + "transport": "http" + } + }, + { + "name": "root", + "type": "task", + "children": [], + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "output": { + "attachment": true + }, + "metadata": { + "case": "json-attachment", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + }, + { + "name": "jest parent span", + "type": "task", + "children": [ + { + "name": "jest child span", + "children": [], + "input": { + "step": "child", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "child" + }, + "metadata": { + "case": "child-span", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + } + ], + "input": { + "phase": "parent", + "testRunId": "" + }, + "output": { + "ok": true, + "phase": "parent" + }, + "metadata": { + "case": "parent-span", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + }, + { + "name": "jest nested parent span", + "type": "task", + "children": [ + { + "name": "jest nested child span", + "children": [ + { + "name": "jest nested grandchild span", + "children": [], + "output": { + "depth": 3 + }, + "metadata": { + "case": "nested-grandchild", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + } + ], + "metadata": { + "case": "nested-child", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + } + ], + "metadata": { + "case": "nested-parent", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + }, + { + "name": "jest current span", + "type": "task", + "children": [], + "output": { + "observedSpanId": "" + }, + "metadata": { + "case": "current-span", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.txt b/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.txt new file mode 100644 index 000000000..dda158638 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-jest/__snapshots__/span-tree.txt @@ -0,0 +1,87 @@ +span_tree: +├── jest basic span [task] +│ input: "What is the capital of France?" +│ output: "Paris" +│ expected: "Paris" +│ metadata: { +│ "case": "basic-span", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "", +│ "transport": "http" +│ } +├── root [task] +│ input: { +│ "transcript": { +│ "content_type": "application/json", +│ "filename": "conversation_transcript.json", +│ "key": "", +│ "type": "braintrust_attachment" +│ }, +│ "type": "chat_completion" +│ } +│ output: { +│ "attachment": true +│ } +│ metadata: { +│ "case": "json-attachment", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +├── jest parent span [task] +│ input: { +│ "phase": "parent", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "parent" +│ } +│ metadata: { +│ "case": "parent-span", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +│ └── jest child span +│ input: { +│ "step": "child", +│ "testRunId": "" +│ } +│ output: { +│ "ok": true, +│ "phase": "child" +│ } +│ metadata: { +│ "case": "child-span", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +├── jest nested parent span [task] +│ metadata: { +│ "case": "nested-parent", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +│ └── jest nested child span +│ metadata: { +│ "case": "nested-child", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +│ └── jest nested grandchild span +│ output: { +│ "depth": 3 +│ } +│ metadata: { +│ "case": "nested-grandchild", +│ "scenario": "test-framework-evals-jest", +│ "testRunId": "" +│ } +└── jest current span [task] + output: { + "observedSpanId": "" + } + metadata: { + "case": "current-span", + "scenario": "test-framework-evals-jest", + "testRunId": "" + } diff --git a/e2e/scenarios/test-framework-evals-jest/scenario.test.ts b/e2e/scenarios/test-framework-evals-jest/scenario.test.ts index fca9f4e57..c1c397d10 100644 --- a/e2e/scenarios/test-framework-evals-jest/scenario.test.ts +++ b/e2e/scenarios/test-framework-evals-jest/scenario.test.ts @@ -4,16 +4,16 @@ import { matchFileSnapshot, resolveFileSnapshotPath, } from "../../helpers/file-snapshot"; -import type { Json } from "../../helpers/normalize"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import type { Json } from "../../helpers/normalize"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; -import { payloadRowsForTestRunId } from "../../helpers/wrapper-contract"; +import { summarizeRequest } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), @@ -35,7 +35,6 @@ test( async () => { await withScenarioHarness( async ({ - payloads, requestCursor, requestsAfter, runNodeScenarioDir, @@ -146,27 +145,9 @@ test( ]), ); - await matchFileSnapshot( - formatJsonFileSnapshot( - [ - basicSpan, - jsonAttachment, - parentSpan, - childSpan, - nestedParent, - nestedChild, - nestedGrandchild, - currentSpan, - ].map((event) => summarizeEvent(event!)) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - - await matchFileSnapshot( - formatJsonFileSnapshot( - payloadRowsForTestRunId(payloads(), testRunId) as Json, - ), - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + await matchSpanTreeSnapshot( + capturedEvents, + resolveFileSnapshotPath(import.meta.url, "span-tree.json"), ); await matchFileSnapshot( diff --git a/e2e/scenarios/test-framework-evals-node/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/test-framework-evals-node/__snapshots__/scenario.test.ts.snap deleted file mode 100644 index b92bd3a7a..000000000 --- a/e2e/scenarios/test-framework-evals-node/__snapshots__/scenario.test.ts.snap +++ /dev/null @@ -1,209 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`test-framework-evals-node captures node:test task spans > log-payloads 1`] = ` -[ - { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0, - }, - "created": "", - "experiment_id": "", - "id": "", - "input": "hello", - "metadata": { - "case": "basic-eval", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metrics": { - "end": 0, - "start": 0, - }, - "output": "echo:hello", - "root_span_id": "", - "scores": { - "pass": 1, - }, - "span_attributes": { - "exec_counter": 0, - "name": "node-test basic eval", - "type": "task", - }, - "span_id": "", - }, - { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0, - }, - "created": "", - "expected": 10, - "experiment_id": "", - "id": "", - "input": { - "value": 5, - }, - "metadata": { - "case": "configured-eval", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metrics": { - "end": 0, - "start": 0, - }, - "output": 10, - "root_span_id": "", - "scores": { - "correctness": 1, - "pass": 1, - }, - "span_attributes": { - "exec_counter": 1, - "name": "node-test configured eval", - "type": "task", - }, - "span_id": "", - "tags": [ - "math", - "configured", - ], - }, - { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0, - }, - "created": "", - "experiment_id": "", - "id": "", - "input": { - "mode": "extra", - }, - "metadata": { - "case": "extra-output", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metrics": { - "end": 0, - "start": 0, - }, - "output": { - "done": true, - "phase": "extra-output", - }, - "root_span_id": "", - "scores": { - "pass": 1, - "quality": 0.95, - }, - "span_attributes": { - "exec_counter": 2, - "name": "node-test extra output", - "type": "task", - }, - "span_id": "", - }, - { - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0, - }, - "created": "", - "experiment_id": "", - "id": "", - "input": "override", - "metadata": { - "case": "name-override", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metrics": { - "end": 0, - "start": 0, - }, - "output": "override", - "root_span_id": "", - "scores": { - "pass": 1, - }, - "span_attributes": { - "exec_counter": 3, - "name": "node-test overridden name", - "type": "task", - }, - "span_id": "", - }, -] -`; - -exports[`test-framework-evals-node captures node:test task spans > span-events 1`] = ` -[ - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "basic-eval", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metric_keys": [], - "name": "node-test basic eval", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task", - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "configured-eval", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metric_keys": [], - "name": "node-test configured eval", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task", - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "extra-output", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metric_keys": [], - "name": "node-test extra output", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task", - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "name-override", - "scenario": "test-framework-evals-node", - "testRunId": "", - }, - "metric_keys": [], - "name": "node-test overridden name", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task", - }, -] -`; diff --git a/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.json b/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.json new file mode 100644 index 000000000..bbae5551b --- /dev/null +++ b/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.json @@ -0,0 +1,78 @@ +{ + "span_tree": [ + { + "name": "node-test basic eval", + "type": "task", + "children": [], + "input": "hello", + "output": "echo:hello", + "scores": { + "pass": 1 + }, + "metadata": { + "case": "basic-eval", + "scenario": "test-framework-evals-node", + "testRunId": "" + } + }, + { + "name": "node-test configured eval", + "type": "task", + "children": [], + "input": { + "value": 5 + }, + "output": 10, + "expected": 10, + "scores": { + "correctness": 1, + "pass": 1 + }, + "tags": [ + "math", + "configured" + ], + "metadata": { + "case": "configured-eval", + "scenario": "test-framework-evals-node", + "testRunId": "" + } + }, + { + "name": "node-test extra output", + "type": "task", + "children": [], + "input": { + "mode": "extra" + }, + "output": { + "done": true, + "phase": "extra-output" + }, + "scores": { + "pass": 1, + "quality": 0.95 + }, + "metadata": { + "case": "extra-output", + "scenario": "test-framework-evals-node", + "testRunId": "" + } + }, + { + "name": "node-test overridden name", + "type": "task", + "children": [], + "input": "override", + "output": "override", + "scores": { + "pass": 1 + }, + "metadata": { + "case": "name-override", + "scenario": "test-framework-evals-node", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.txt b/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.txt new file mode 100644 index 000000000..887236798 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-node/__snapshots__/span-tree.txt @@ -0,0 +1,59 @@ +span_tree: +├── node-test basic eval [task] +│ input: "hello" +│ output: "echo:hello" +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "basic-eval", +│ "scenario": "test-framework-evals-node", +│ "testRunId": "" +│ } +├── node-test configured eval [task] +│ input: { +│ "value": 5 +│ } +│ output: 10 +│ expected: 10 +│ scores: { +│ "correctness": 1, +│ "pass": 1 +│ } +│ tags: [ +│ "math", +│ "configured" +│ ] +│ metadata: { +│ "case": "configured-eval", +│ "scenario": "test-framework-evals-node", +│ "testRunId": "" +│ } +├── node-test extra output [task] +│ input: { +│ "mode": "extra" +│ } +│ output: { +│ "done": true, +│ "phase": "extra-output" +│ } +│ scores: { +│ "pass": 1, +│ "quality": 0.95 +│ } +│ metadata: { +│ "case": "extra-output", +│ "scenario": "test-framework-evals-node", +│ "testRunId": "" +│ } +└── node-test overridden name [task] + input: "override" + output: "override" + scores: { + "pass": 1 + } + metadata: { + "case": "name-override", + "scenario": "test-framework-evals-node", + "testRunId": "" + } diff --git a/e2e/scenarios/test-framework-evals-node/scenario.test.ts b/e2e/scenarios/test-framework-evals-node/scenario.test.ts index 3e53b52cb..964f9076f 100644 --- a/e2e/scenarios/test-framework-evals-node/scenario.test.ts +++ b/e2e/scenarios/test-framework-evals-node/scenario.test.ts @@ -1,20 +1,21 @@ import { expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { - payloadRowsForTestRunId, - summarizeWrapperContract, -} from "../../helpers/wrapper-contract"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); const TIMEOUT_MS = 90_000; +const spanTreeSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "span-tree.json", +); test( "test-framework-evals-node captures node:test task spans", @@ -23,7 +24,7 @@ test( }, async () => { await withScenarioHarness( - async ({ payloads, runScenarioDir, testRunEvents, testRunId }) => { + async ({ runScenarioDir, testRunEvents, testRunId }) => { await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); const capturedEvents = testRunEvents(); @@ -79,24 +80,7 @@ test( expect(nameOverride?.span.name).toBe("node-test overridden name"); - expect( - normalizeForSnapshot( - [basicEval, configuredEval, extraOutput, nameOverride].map( - (event) => - summarizeWrapperContract(event!, [ - "case", - "scenario", - "testRunId", - ]), - ) as Json, - ), - ).toMatchSnapshot("span-events"); - - expect( - normalizeForSnapshot( - payloadRowsForTestRunId(payloads(), testRunId) as Json, - ), - ).toMatchSnapshot("log-payloads"); + await matchSpanTreeSnapshot(capturedEvents, spanTreeSnapshotPath); }, ); }, diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.log-payloads.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.log-payloads.json deleted file mode 100644 index 2c6eb9196..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.log-payloads.json +++ /dev/null @@ -1,174 +0,0 @@ -[ - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "Module.runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "simple-pass" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 0, - "name": "vitest simple pass", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "Module.runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "expected": 10, - "experiment_id": "", - "id": "", - "input": { - "value": 5 - }, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "configured-span", - "result": 10 - }, - "root_span_id": "", - "scores": { - "correctness": 1, - "pass": 1, - "quality": 0.9 - }, - "span_attributes": { - "exec_counter": 1, - "name": "vitest configured span", - "type": "task" - }, - "span_id": "", - "tags": [ - "math", - "configured" - ] - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "Module.runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-alpha" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 2, - "name": "vitest concurrent alpha", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "Module.runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-beta" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 3, - "name": "vitest concurrent beta", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "Module.runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at Module.runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)", - "experiment_id": "", - "id": "", - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "expected-failure" - }, - "root_span_id": "", - "scores": { - "pass": 0 - }, - "span_attributes": { - "exec_counter": 4, - "name": "vitest expected failure", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-events.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-events.json deleted file mode 100644 index 3223aad29..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-events.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest simple pass", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest configured span", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent alpha", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent beta", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest expected failure", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.json new file mode 100644 index 000000000..fa83fb077 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.json @@ -0,0 +1,96 @@ +{ + "span_tree": [ + { + "name": "vitest simple pass", + "type": "task", + "children": [], + "output": { + "phase": "simple-pass" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "simple-pass", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest configured span", + "type": "task", + "children": [], + "input": { + "value": 5 + }, + "output": { + "phase": "configured-span", + "result": 10 + }, + "expected": 10, + "scores": { + "correctness": 1, + "pass": 1, + "quality": 0.9 + }, + "tags": [ + "math", + "configured" + ], + "metadata": { + "case": "configured-span", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent alpha", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-alpha" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-alpha", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent beta", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-beta" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-beta", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest expected failure", + "type": "task", + "children": [], + "output": { + "phase": "expected-failure" + }, + "scores": { + "pass": 0 + }, + "metadata": { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + }, + "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at Module.runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" + } + ] +} diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.txt b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.txt new file mode 100644 index 000000000..509669bb4 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v2.span-tree.txt @@ -0,0 +1,73 @@ +span_tree: +├── vitest simple pass [task] +│ output: { +│ "phase": "simple-pass" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "simple-pass", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest configured span [task] +│ input: { +│ "value": 5 +│ } +│ output: { +│ "phase": "configured-span", +│ "result": 10 +│ } +│ expected: 10 +│ scores: { +│ "correctness": 1, +│ "pass": 1, +│ "quality": 0.9 +│ } +│ tags: [ +│ "math", +│ "configured" +│ ] +│ metadata: { +│ "case": "configured-span", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent alpha [task] +│ output: { +│ "phase": "concurrent-alpha" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-alpha", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent beta [task] +│ output: { +│ "phase": "concurrent-beta" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-beta", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +└── vitest expected failure [task] + output: { + "phase": "expected-failure" + } + scores: { + "pass": 0 + } + metadata: { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + error: "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at Module.runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.log-payloads.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.log-payloads.json deleted file mode 100644 index a37c9b389..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.log-payloads.json +++ /dev/null @@ -1,174 +0,0 @@ -[ - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "simple-pass" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 0, - "name": "vitest simple pass", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "expected": 10, - "experiment_id": "", - "id": "", - "input": { - "value": 5 - }, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "configured-span", - "result": 10 - }, - "root_span_id": "", - "scores": { - "correctness": 1, - "pass": 1, - "quality": 0.9 - }, - "span_attributes": { - "exec_counter": 1, - "name": "vitest configured span", - "type": "task" - }, - "span_id": "", - "tags": [ - "math", - "configured" - ] - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-alpha" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 2, - "name": "vitest concurrent alpha", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-beta" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 3, - "name": "vitest concurrent beta", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)", - "experiment_id": "", - "id": "", - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "expected-failure" - }, - "root_span_id": "", - "scores": { - "pass": 0 - }, - "span_attributes": { - "exec_counter": 4, - "name": "vitest expected failure", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-events.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-events.json deleted file mode 100644 index 3223aad29..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-events.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest simple pass", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest configured span", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent alpha", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent beta", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest expected failure", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.json new file mode 100644 index 000000000..23e801424 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.json @@ -0,0 +1,96 @@ +{ + "span_tree": [ + { + "name": "vitest simple pass", + "type": "task", + "children": [], + "output": { + "phase": "simple-pass" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "simple-pass", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest configured span", + "type": "task", + "children": [], + "input": { + "value": 5 + }, + "output": { + "phase": "configured-span", + "result": 10 + }, + "expected": 10, + "scores": { + "correctness": 1, + "pass": 1, + "quality": 0.9 + }, + "tags": [ + "math", + "configured" + ], + "metadata": { + "case": "configured-span", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent alpha", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-alpha" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-alpha", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent beta", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-beta" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-beta", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest expected failure", + "type": "task", + "children": [], + "output": { + "phase": "expected-failure" + }, + "scores": { + "pass": 0 + }, + "metadata": { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + }, + "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" + } + ] +} diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.txt b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.txt new file mode 100644 index 000000000..6ab1bf413 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v3.span-tree.txt @@ -0,0 +1,73 @@ +span_tree: +├── vitest simple pass [task] +│ output: { +│ "phase": "simple-pass" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "simple-pass", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest configured span [task] +│ input: { +│ "value": 5 +│ } +│ output: { +│ "phase": "configured-span", +│ "result": 10 +│ } +│ expected: 10 +│ scores: { +│ "correctness": 1, +│ "pass": 1, +│ "quality": 0.9 +│ } +│ tags: [ +│ "math", +│ "configured" +│ ] +│ metadata: { +│ "case": "configured-span", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent alpha [task] +│ output: { +│ "phase": "concurrent-alpha" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-alpha", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent beta [task] +│ output: { +│ "phase": "concurrent-beta" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-beta", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +└── vitest expected failure [task] + output: { + "phase": "expected-failure" + } + scores: { + "pass": 0 + } + metadata: { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + error: "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.log-payloads.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.log-payloads.json deleted file mode 100644 index a37c9b389..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.log-payloads.json +++ /dev/null @@ -1,174 +0,0 @@ -[ - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "simple-pass" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 0, - "name": "vitest simple pass", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "expected": 10, - "experiment_id": "", - "id": "", - "input": { - "value": 5 - }, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "configured-span", - "result": 10 - }, - "root_span_id": "", - "scores": { - "correctness": 1, - "pass": 1, - "quality": 0.9 - }, - "span_attributes": { - "exec_counter": 1, - "name": "vitest configured span", - "type": "task" - }, - "span_id": "", - "tags": [ - "math", - "configured" - ] - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-alpha" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 2, - "name": "vitest concurrent alpha", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "experiment_id": "", - "id": "", - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "concurrent-beta" - }, - "root_span_id": "", - "scores": { - "pass": 1 - }, - "span_attributes": { - "exec_counter": 3, - "name": "vitest concurrent beta", - "type": "task" - }, - "span_id": "" - }, - { - "context": { - "caller_filename": "/js/src/wrappers/shared/traced-eval.ts", - "caller_functionname": "runTracedEval", - "caller_lineno": 0 - }, - "created": "", - "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)", - "experiment_id": "", - "id": "", - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "phase": "expected-failure" - }, - "root_span_id": "", - "scores": { - "pass": 0 - }, - "span_attributes": { - "exec_counter": 4, - "name": "vitest expected failure", - "type": "task" - }, - "span_id": "" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-events.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-events.json deleted file mode 100644 index 3223aad29..000000000 --- a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-events.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "simple-pass", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest simple pass", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "case": "configured-span", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest configured span", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-alpha", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent alpha", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "concurrent-beta", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest concurrent beta", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": true, - "metadata": { - "case": "expected-failure", - "scenario": "test-framework-evals-vitest", - "testRunId": "" - }, - "metric_keys": [], - "name": "vitest expected failure", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - } -] diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.json b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.json new file mode 100644 index 000000000..23e801424 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.json @@ -0,0 +1,96 @@ +{ + "span_tree": [ + { + "name": "vitest simple pass", + "type": "task", + "children": [], + "output": { + "phase": "simple-pass" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "simple-pass", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest configured span", + "type": "task", + "children": [], + "input": { + "value": 5 + }, + "output": { + "phase": "configured-span", + "result": 10 + }, + "expected": 10, + "scores": { + "correctness": 1, + "pass": 1, + "quality": 0.9 + }, + "tags": [ + "math", + "configured" + ], + "metadata": { + "case": "configured-span", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent alpha", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-alpha" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-alpha", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest concurrent beta", + "type": "task", + "children": [], + "output": { + "phase": "concurrent-beta" + }, + "scores": { + "pass": 1 + }, + "metadata": { + "case": "concurrent-beta", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + }, + { + "name": "vitest expected failure", + "type": "task", + "children": [], + "output": { + "phase": "expected-failure" + }, + "scores": { + "pass": 0 + }, + "metadata": { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + }, + "error": "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" + } + ] +} diff --git a/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.txt b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.txt new file mode 100644 index 000000000..6ab1bf413 --- /dev/null +++ b/e2e/scenarios/test-framework-evals-vitest/__snapshots__/v4.1.span-tree.txt @@ -0,0 +1,73 @@ +span_tree: +├── vitest simple pass [task] +│ output: { +│ "phase": "simple-pass" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "simple-pass", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest configured span [task] +│ input: { +│ "value": 5 +│ } +│ output: { +│ "phase": "configured-span", +│ "result": 10 +│ } +│ expected: 10 +│ scores: { +│ "correctness": 1, +│ "pass": 1, +│ "quality": 0.9 +│ } +│ tags: [ +│ "math", +│ "configured" +│ ] +│ metadata: { +│ "case": "configured-span", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent alpha [task] +│ output: { +│ "phase": "concurrent-alpha" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-alpha", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +├── vitest concurrent beta [task] +│ output: { +│ "phase": "concurrent-beta" +│ } +│ scores: { +│ "pass": 1 +│ } +│ metadata: { +│ "case": "concurrent-beta", +│ "scenario": "test-framework-evals-vitest", +│ "testRunId": "" +│ } +└── vitest expected failure [task] + output: { + "phase": "expected-failure" + } + scores: { + "pass": 0 + } + metadata: { + "case": "expected-failure", + "scenario": "test-framework-evals-vitest", + "testRunId": "" + } + error: "expected 'wrong' to be 'right' // Object.is equality\n\nAssertionError: expected 'wrong' to be 'right' // Object.is equality\n at /e2e/scenarios/test-framework-evals-vitest/runner.case.ts:0:0\n at Object.fn (/js/src/wrappers/vitest/wrapper.ts:0:0)\n at args.experiment.traced.name (/js/src/wrappers/shared/traced-eval.ts:0:0)\n at /js/src/logger.ts:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/src/logger.ts:0:0)\n at withCurrent (/js/src/logger.ts:0:0)\n at /js/src/logger.ts:0:0\n at runCatchFinally (/js/src/util.ts:0:0)\n at Experiment.traced (/js/src/logger.ts:0:0)" diff --git a/e2e/scenarios/test-framework-evals-vitest/scenario.test.ts b/e2e/scenarios/test-framework-evals-vitest/scenario.test.ts index e7b5f8052..969c9965b 100644 --- a/e2e/scenarios/test-framework-evals-vitest/scenario.test.ts +++ b/e2e/scenarios/test-framework-evals-vitest/scenario.test.ts @@ -1,21 +1,12 @@ import { expect, test } from "vitest"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; -import type { Json } from "../../helpers/normalize"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { prepareScenarioDir, - readInstalledPackageVersion, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { - payloadRowsForTestRunId, - summarizeWrapperContract, -} from "../../helpers/wrapper-contract"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), @@ -41,7 +32,7 @@ for (const scenario of scenarios) { }, async () => { await withScenarioHarness( - async ({ payloads, runScenarioDir, testRunEvents, testRunId }) => { + async ({ runScenarioDir, testRunEvents, testRunId }) => { await runScenarioDir({ entry: scenario.entry, scenarioDir, @@ -110,35 +101,11 @@ for (const scenario of scenarios) { pass: 0, }); - await matchFileSnapshot( - formatJsonFileSnapshot( - [ - simplePass, - configured, - concurrentAlpha, - concurrentBeta, - expectedFailure, - ].map((event) => - summarizeWrapperContract(event!, [ - "case", - "scenario", - "testRunId", - ]), - ) as Json, - ), - resolveFileSnapshotPath( - import.meta.url, - `${scenario.label}.span-events.json`, - ), - ); - - await matchFileSnapshot( - formatJsonFileSnapshot( - payloadRowsForTestRunId(payloads(), testRunId) as Json, - ), + await matchSpanTreeSnapshot( + capturedEvents, resolveFileSnapshotPath( import.meta.url, - `${scenario.label}.log-payloads.json`, + `${scenario.label}.span-tree.json`, ), ); }, diff --git a/e2e/scenarios/trace-context-and-continuation/__snapshots__/late-update-payloads.json b/e2e/scenarios/trace-context-and-continuation/__snapshots__/late-update-payloads.json new file mode 100644 index 000000000..76b81d216 --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/__snapshots__/late-update-payloads.json @@ -0,0 +1,44 @@ +[ + { + "context": { + "caller_filename": "/e2e/scenarios/trace-context-and-continuation/scenario.ts", + "caller_functionname": "main", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "kind": "late-update", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "late-update", + "type": "task" + }, + "span_id": "" + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metadata": { + "kind": "late-update", + "patched": true, + "testRunId": "" + }, + "output": { + "state": "updated" + }, + "project_id": "", + "root_span_id": "", + "span_id": "" + } +] diff --git a/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap deleted file mode 100644 index 5e1210f85..000000000 --- a/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap +++ /dev/null @@ -1,134 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`trace-context-and-continuation supports reattachment and late span updates > late-update-payloads 1`] = ` -[ - { - "context": { - "caller_filename": "/e2e/scenarios/trace-context-and-continuation/scenario.ts", - "caller_functionname": "main", - "caller_lineno": 0, - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "kind": "late-update", - "testRunId": "", - }, - "metrics": { - "end": 0, - "start": 0, - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "late-update", - "type": "task", - }, - "span_id": "", - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "kind": "late-update", - "patched": true, - "testRunId": "", - }, - "output": { - "state": "updated", - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - }, -] -`; - -exports[`trace-context-and-continuation supports reattachment and late span updates > span-events 1`] = ` -[ - { - "error": null, - "input": null, - "metadata": { - "scenario": "trace-context-and-continuation", - "testRunId": "", - }, - "name": "context-root", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "context-root", - "type": "task", - }, - "span_id": "", - "span_parents": null, - }, - { - "error": null, - "input": null, - "metadata": { - "kind": "current-child", - "testRunId": "", - }, - "name": "current-child", - "output": { - "source": "withCurrent", - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "current-child", - }, - "span_id": "", - "span_parents": [ - "", - ], - }, - { - "error": null, - "input": null, - "metadata": { - "kind": "reattached-child", - "testRunId": "", - }, - "name": "reattached-child", - "output": { - "resumed": true, - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "reattached-child", - }, - "span_id": "", - "span_parents": [ - "", - ], - }, - { - "error": null, - "input": null, - "metadata": { - "kind": "late-update", - "patched": true, - "testRunId": "", - }, - "name": "late-update", - "output": { - "state": "updated", - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "late-update", - "type": "task", - }, - "span_id": "", - "span_parents": null, - }, -] -`; diff --git a/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.json b/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.json new file mode 100644 index 000000000..86b75db77 --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.json @@ -0,0 +1,49 @@ +{ + "span_tree": [ + { + "name": "context-root", + "type": "task", + "children": [ + { + "name": "current-child", + "children": [], + "output": { + "source": "withCurrent" + }, + "metadata": { + "kind": "current-child", + "testRunId": "" + } + }, + { + "name": "reattached-child", + "children": [], + "output": { + "resumed": true + }, + "metadata": { + "kind": "reattached-child", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "trace-context-and-continuation", + "testRunId": "" + } + }, + { + "name": "late-update", + "type": "task", + "children": [], + "output": { + "state": "updated" + }, + "metadata": { + "kind": "late-update", + "patched": true, + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.txt b/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.txt new file mode 100644 index 000000000..f6ab8563a --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/__snapshots__/span-tree.txt @@ -0,0 +1,31 @@ +span_tree: +├── context-root [task] +│ metadata: { +│ "scenario": "trace-context-and-continuation", +│ "testRunId": "" +│ } +│ ├── current-child +│ │ output: { +│ │ "source": "withCurrent" +│ │ } +│ │ metadata: { +│ │ "kind": "current-child", +│ │ "testRunId": "" +│ │ } +│ └── reattached-child +│ output: { +│ "resumed": true +│ } +│ metadata: { +│ "kind": "reattached-child", +│ "testRunId": "" +│ } +└── late-update [task] + output: { + "state": "updated" + } + metadata: { + "kind": "late-update", + "patched": true, + "testRunId": "" + } diff --git a/e2e/scenarios/trace-context-and-continuation/scenario.test.ts b/e2e/scenarios/trace-context-and-continuation/scenario.test.ts index 9c17133e6..f386798e2 100644 --- a/e2e/scenarios/trace-context-and-continuation/scenario.test.ts +++ b/e2e/scenarios/trace-context-and-continuation/scenario.test.ts @@ -1,16 +1,29 @@ import { expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + formatJsonFileSnapshot, + matchFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import type { Json } from "../../helpers/normalize"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeEvent } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); +const spanTreeSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "span-tree.json", +); +const lateUpdatePayloadsSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "late-update-payloads.json", +); test("trace-context-and-continuation supports reattachment and late span updates", async () => { await withScenarioHarness( @@ -42,18 +55,7 @@ test("trace-context-and-continuation supports reattachment and late span updates state: "updated", }); - expect( - normalizeForSnapshot( - [ - "context-root", - "current-child", - "reattached-child", - "late-update", - ].map((name) => - summarizeEvent(findLatestSpan(capturedEvents, name)!), - ) as Json, - ), - ).toMatchSnapshot("span-events"); + await matchSpanTreeSnapshot(capturedEvents, spanTreeSnapshotPath); const mutationRows = payloads() .flatMap((payload) => payload.rows) @@ -70,8 +72,9 @@ test("trace-context-and-continuation supports reattachment and late span updates ); }); - expect(normalizeForSnapshot(mutationRows as Json)).toMatchSnapshot( - "late-update-payloads", + await matchFileSnapshot( + formatJsonFileSnapshot(mutationRows as Json), + lateUpdatePayloadsSnapshotPath, ); }, ); diff --git a/e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/trace-primitives-basic/__snapshots__/request-flow.json similarity index 53% rename from e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap rename to e2e/scenarios/trace-primitives-basic/__snapshots__/request-flow.json index afe8ced5c..6a2c3fdb7 100644 --- a/e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap +++ b/e2e/scenarios/trace-primitives-basic/__snapshots__/request-flow.json @@ -1,6 +1,3 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`trace-primitives-basic collects a minimal manual trace tree > request-flow 1`] = ` [ { "headers": null, @@ -8,21 +5,21 @@ exports[`trace-primitives-basic collects a minimal manual trace tree > request-f "method": "POST", "path": "/api/apikey/login", "query": null, - "rawBody": null, + "rawBody": null }, { "headers": null, "jsonBody": { "org_id": "mock-org-id", - "project_name": "", + "project_name": "" }, "method": "POST", "path": "/api/project/register", "query": null, "rawBody": { "org_id": "mock-org-id", - "project_name": "", - }, + "project_name": "" + } }, { "headers": null, @@ -30,7 +27,7 @@ exports[`trace-primitives-basic collects a minimal manual trace tree > request-f "method": "GET", "path": "/version", "query": null, - "rawBody": null, + "rawBody": null }, { "headers": null, @@ -41,112 +38,100 @@ exports[`trace-primitives-basic collects a minimal manual trace tree > request-f "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "main", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", "id": "", "input": { "scenario": "trace-primitives-basic", - "testRunId": "", + "testRunId": "" }, "log_id": "g", "metadata": { "scenario": "trace-primitives-basic", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "output": { - "status": "ok", + "status": "ok" }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 0, "name": "trace-primitives-root", - "type": "task", + "type": "task" }, - "span_id": "", + "span_id": "" }, { "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "logger.traced.name", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", "id": "", "input": { "step": "child", - "testRunId": "", + "testRunId": "" }, "log_id": "g", "metadata": { "kind": "basic-child", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "output": { - "ok": true, + "ok": true }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 1, - "name": "basic-child", + "name": "basic-child" }, "span_id": "", "span_parents": [ - "", - ], + "" + ] }, { "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "logger.traced.name", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", - "error": "basic boom - -Error: basic boom - at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) - at /js/dist/index.mjs:0:0 - at AsyncLocalStorage.run (node::0:0) - at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0) - at withCurrent (/js/dist/index.mjs:0:0) - at /js/dist/index.mjs:0:0 - at runCatchFinally (/js/dist/index.mjs:0:0) - at Logger.traced (/js/dist/index.mjs:0:0) - at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) - at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", + "error": "basic boom\n\nError: basic boom\n at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at Logger.traced (/js/dist/index.mjs:0:0)\n at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", "id": "", "log_id": "g", "metadata": { "kind": "basic-error", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 2, - "name": "basic-error", + "name": "basic-error" }, "span_id": "", "span_parents": [ - "", - ], - }, - ], + "" + ] + } + ] }, "method": "POST", "path": "/logs3", @@ -158,184 +143,100 @@ Error: basic boom "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "main", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", "id": "", "input": { "scenario": "trace-primitives-basic", - "testRunId": "", + "testRunId": "" }, "log_id": "g", "metadata": { "scenario": "trace-primitives-basic", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "output": { - "status": "ok", + "status": "ok" }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 0, "name": "trace-primitives-root", - "type": "task", + "type": "task" }, - "span_id": "", + "span_id": "" }, { "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "logger.traced.name", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", "id": "", "input": { "step": "child", - "testRunId": "", + "testRunId": "" }, "log_id": "g", "metadata": { "kind": "basic-child", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "output": { - "ok": true, + "ok": true }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 1, - "name": "basic-child", + "name": "basic-child" }, "span_id": "", "span_parents": [ - "", - ], + "" + ] }, { "context": { "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", "caller_functionname": "logger.traced.name", - "caller_lineno": 0, + "caller_lineno": 0 }, "created": "", - "error": "basic boom - -Error: basic boom - at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) - at /js/dist/index.mjs:0:0 - at AsyncLocalStorage.run (node::0:0) - at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0) - at withCurrent (/js/dist/index.mjs:0:0) - at /js/dist/index.mjs:0:0 - at runCatchFinally (/js/dist/index.mjs:0:0) - at Logger.traced (/js/dist/index.mjs:0:0) - at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) - at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", + "error": "basic boom\n\nError: basic boom\n at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at Logger.traced (/js/dist/index.mjs:0:0)\n at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", "id": "", "log_id": "g", "metadata": { "kind": "basic-error", - "testRunId": "", + "testRunId": "" }, "metrics": { "end": 0, - "start": 0, + "start": 0 }, "project_id": "", "root_span_id": "", "span_attributes": { "exec_counter": 2, - "name": "basic-error", + "name": "basic-error" }, "span_id": "", "span_parents": [ - "", - ], - }, - ], - }, - }, -] -`; - -exports[`trace-primitives-basic collects a minimal manual trace tree > span-events 1`] = ` -[ - { - "error": null, - "input": { - "scenario": "trace-primitives-basic", - "testRunId": "", - }, - "metadata": { - "scenario": "trace-primitives-basic", - "testRunId": "", - }, - "name": "trace-primitives-root", - "output": { - "status": "ok", - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "trace-primitives-root", - "type": "task", - }, - "span_id": "", - "span_parents": null, - }, - { - "error": null, - "input": { - "step": "child", - "testRunId": "", - }, - "metadata": { - "kind": "basic-child", - "testRunId": "", - }, - "name": "basic-child", - "output": { - "ok": true, - }, - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "basic-child", - }, - "span_id": "", - "span_parents": [ - "", - ], - }, - { - "error": "basic boom", - "input": null, - "metadata": { - "kind": "basic-error", - "testRunId": "", - }, - "name": "basic-error", - "output": null, - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "basic-error", - }, - "span_id": "", - "span_parents": [ - "", - ], - }, + "" + ] + } + ] + } + } ] -`; diff --git a/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.json b/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.json new file mode 100644 index 000000000..34a4e6d28 --- /dev/null +++ b/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.json @@ -0,0 +1,45 @@ +{ + "span_tree": [ + { + "name": "trace-primitives-root", + "type": "task", + "children": [ + { + "name": "basic-child", + "children": [], + "input": { + "step": "child", + "testRunId": "" + }, + "output": { + "ok": true + }, + "metadata": { + "kind": "basic-child", + "testRunId": "" + } + }, + { + "name": "basic-error", + "children": [], + "metadata": { + "kind": "basic-error", + "testRunId": "" + }, + "error": "basic boom\n\nError: basic boom\n at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at Logger.traced (/js/dist/index.mjs:0:0)\n at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at runMain (/e2e/helpers/scenario-runtime.ts:0:0)" + } + ], + "input": { + "scenario": "trace-primitives-basic", + "testRunId": "" + }, + "output": { + "status": "ok" + }, + "metadata": { + "scenario": "trace-primitives-basic", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.txt b/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.txt new file mode 100644 index 000000000..3bb8e514b --- /dev/null +++ b/e2e/scenarios/trace-primitives-basic/__snapshots__/span-tree.txt @@ -0,0 +1,31 @@ +span_tree: +└── trace-primitives-root [task] + input: { + "scenario": "trace-primitives-basic", + "testRunId": "" + } + output: { + "status": "ok" + } + metadata: { + "scenario": "trace-primitives-basic", + "testRunId": "" + } + ├── basic-child + │ input: { + │ "step": "child", + │ "testRunId": "" + │ } + │ output: { + │ "ok": true + │ } + │ metadata: { + │ "kind": "basic-child", + │ "testRunId": "" + │ } + └── basic-error + metadata: { + "kind": "basic-error", + "testRunId": "" + } + error: "basic boom\n\nError: basic boom\n at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at /js/dist/index.mjs:0:0\n at AsyncLocalStorage.run (node::0:0)\n at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0)\n at withCurrent (/js/dist/index.mjs:0:0)\n at /js/dist/index.mjs:0:0\n at runCatchFinally (/js/dist/index.mjs:0:0)\n at Logger.traced (/js/dist/index.mjs:0:0)\n at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0)\n at runMain (/e2e/helpers/scenario-runtime.ts:0:0)" diff --git a/e2e/scenarios/trace-primitives-basic/scenario.test.ts b/e2e/scenarios/trace-primitives-basic/scenario.test.ts index f2e8fd518..4eb24f5c7 100644 --- a/e2e/scenarios/trace-primitives-basic/scenario.test.ts +++ b/e2e/scenarios/trace-primitives-basic/scenario.test.ts @@ -1,16 +1,30 @@ import { expect, test } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + formatJsonFileSnapshot, + matchFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import type { Json } from "../../helpers/normalize"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { findLatestSpan } from "../../helpers/trace-selectors"; -import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; +import { summarizeRequest } from "../../helpers/trace-summary"; const scenarioDir = await prepareScenarioDir({ scenarioDir: resolveScenarioDir(import.meta.url), }); +const spanTreeSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "span-tree.json", +); +const requestFlowSnapshotPath = resolveFileSnapshotPath( + import.meta.url, + "request-flow.json", +); test("trace-primitives-basic collects a minimal manual trace tree", async () => { await withScenarioHarness( @@ -32,13 +46,7 @@ test("trace-primitives-basic collects a minimal manual trace tree", async () => expect(error?.span.parentIds).toEqual([root?.span.id ?? ""]); expect(root?.span.rootId).toBe(root?.span.id); - expect( - normalizeForSnapshot( - ["trace-primitives-root", "basic-child", "basic-error"].map((name) => - summarizeEvent(findLatestSpan(capturedEvents, name)!), - ) as Json, - ), - ).toMatchSnapshot("span-events"); + await matchSpanTreeSnapshot(capturedEvents, spanTreeSnapshotPath); const requests = requestsAfter( cursor, @@ -49,15 +57,16 @@ test("trace-primitives-basic collects a minimal manual trace tree", async () => request.path === "/logs3", ); - expect( - normalizeForSnapshot( + await matchFileSnapshot( + formatJsonFileSnapshot( requests.map((request) => summarizeRequest(request, { normalizeJsonRawBody: true, }), ) as Json, ), - ).toMatchSnapshot("request-flow"); + requestFlowSnapshotPath, + ); }, ); }); diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json deleted file mode 100644 index 371b3e6b4..000000000 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/log-payloads.json +++ /dev/null @@ -1,1800 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "wrap-langchain-js-traces", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "langchain-wrapper-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "invoke", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "langchain-invoke-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with exactly OK.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 24, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 24, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 24, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - }, - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "chain", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "langchain-chain-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "word": "PARIS" - }, - "log_id": "g", - "metadata": { - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "metadata": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain_core", - "runnables", - "RunnableSequence" - ], - "kwargs": { - "first": { - "id": [ - "langchain_core", - "prompts", - "chat", - "ChatPromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "messages": [ - { - "id": [ - "langchain_core", - "prompts", - "chat", - "HumanMessagePromptTemplate" - ], - "kwargs": { - "prompt": { - "id": [ - "langchain_core", - "prompts", - "prompt", - "PromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "template": "Reply with the single word {word} and nothing else.", - "template_format": "f-string" - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "last": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "PARIS", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": 2, - "promptTokens": 18, - "totalTokens": 20 - }, - "usage": { - "completion_tokens": 2, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens": 18, - "prompt_tokens_details": { - "audio_tokens": 0, - "cached_tokens": 0 - }, - "total_tokens": 20 - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": 0, - "cache_read": 0 - }, - "input_tokens": 18, - "output_token_details": { - "audio": 0, - "reasoning": 0 - }, - "output_tokens": 2, - "total_tokens": 20 - } - }, - "lc": 1, - "type": "constructor" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "RunnableSequence", - "type": "task" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "word": "PARIS" - }, - "log_id": "g", - "metadata": { - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "metadata": {}, - "name": "ChatPromptTemplate", - "parent_run_id": "", - "run_id": "", - "run_type": "prompt", - "serialized": { - "id": [ - "langchain_core", - "prompts", - "chat", - "ChatPromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "messages": [ - { - "id": [ - "langchain_core", - "prompts", - "chat", - "HumanMessagePromptTemplate" - ], - "kwargs": { - "prompt": { - "id": [ - "langchain_core", - "prompts", - "prompt", - "PromptTemplate" - ], - "kwargs": { - "input_variables": [ - "word" - ], - "template": "Reply with the single word {word} and nothing else.", - "template_format": "f-string" - }, - "lc": 1, - "type": "constructor" - } - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "tags": [ - "seq:step:1" - ] - }, - "metrics": { - "end": 0, - "start": 0 - }, - "output": { - "id": [ - "langchain_core", - "prompt_values", - "ChatPromptValue" - ], - "kwargs": { - "messages": [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with the single word PARIS and nothing else.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "ChatPromptTemplate", - "type": "task" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Reply with the single word PARIS and nothing else.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 32, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "parent_run_id": "", - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [ - "seq:step:2" - ] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - }, - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "langchain-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Count from 1 to 3 and include the words one two three.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "stream": true, - "stream_options": { - "include_usage": true - }, - "temperature": 0 - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 32, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": {}, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 32, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "streaming": true, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "time_to_first_token": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "completion": 0, - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "prompt": 0, - "service_tier": "default", - "system_fingerprint": "" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessageChunk" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "completion": 0, - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "prompt": 0, - "service_tier": "default", - "system_fingerprint": "", - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_call_chunks": [], - "tool_calls": [], - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "langchain-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "tool_calls" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": { - "tool_calls": [ - { - "function": { - "arguments": "{\"location\":\"Paris, France\"}", - "name": "get_weather" - }, - "id": null, - "type": "function" - } - ] - }, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "tool_calls", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - }, - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_calls": [ - { - "args": { - "location": "Paris, France" - }, - "id": null, - "name": "get_weather", - "type": "tool_call" - } - ], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool-result", - "testRunId": "" - }, - "metrics": { - "end": 0, - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "langchain-tool-result-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "What is 127 multiplied by 49? Use the calculate tool.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "tool_calls" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": { - "tool_calls": [ - { - "function": { - "arguments": "{\"operation\":\"multiply\",\"a\":127,\"b\":49}", - "name": "calculate" - }, - "id": null, - "type": "function" - } - ] - }, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "tool_calls", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - }, - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_calls": [ - { - "args": { - "a": 127, - "b": 49, - "operation": "multiply" - }, - "id": null, - "name": "calculate", - "type": "tool_call" - } - ], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": { - "caller_filename": "/integrations/langchain-js/dist/index.js", - "caller_functionname": "BraintrustCallbackHandler.startSpan", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - [ - { - "id": [ - "langchain_core", - "messages", - "HumanMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "What is 127 multiplied by 49? Use the calculate tool.", - "response_metadata": {} - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "invalid_tool_calls": [], - "response_metadata": {}, - "tool_calls": [ - { - "args": { - "a": 127, - "b": 49, - "operation": "multiply" - }, - "id": null, - "name": "calculate", - "type": "tool_call" - } - ] - }, - "lc": 1, - "type": "constructor" - }, - { - "id": [ - "langchain_core", - "messages", - "ToolMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "6223", - "response_metadata": {}, - "tool_call_id": "" - }, - "lc": 1, - "type": "constructor" - } - ] - ], - "log_id": "g", - "metadata": { - "batch_size": 1, - "braintrust": { - "integration_name": "langchain-js", - "integration_version": "0.2.0", - "sdk_language": "javascript" - }, - "invocation_params": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "stream": false, - "temperature": 0, - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "metadata": { - "ls_integration": "langchain_chat_model", - "ls_max_tokens": 128, - "ls_model_name": "gpt-4o-mini-2024-07-18", - "ls_model_type": "chat", - "ls_provider": "openai", - "ls_temperature": 0, - "versions": { - "@langchain/core": "", - "@langchain/openai": "" - } - }, - "model": "gpt-4o-mini-2024-07-18", - "options": { - "tools": [ - { - "function": { - "description": "Perform a mathematical calculation", - "name": "calculate", - "parameters": { - "$schema": "http://json-schema.org/draft-07/schema#", - "additionalProperties": false, - "properties": { - "a": { - "type": "number" - }, - "b": { - "type": "number" - }, - "operation": { - "enum": [ - "add", - "subtract", - "multiply", - "divide" - ], - "type": "string" - } - }, - "required": [ - "operation", - "a", - "b" - ], - "type": "object" - } - }, - "type": "function" - } - ] - }, - "run_id": "", - "serialized": { - "id": [ - "langchain", - "chat_models", - "openai", - "ChatOpenAI" - ], - "kwargs": { - "max_tokens": 128, - "model": "gpt-4o-mini-2024-07-18", - "openai_api_key": { - "id": [ - "OPENAI_API_KEY" - ], - "lc": 1, - "type": "secret" - }, - "temperature": 0 - }, - "lc": 1, - "type": "constructor" - }, - "tags": [] - }, - "metrics": { - "completion_tokens": "", - "end": 0, - "prompt_cached_tokens": "", - "prompt_tokens": "", - "start": 0, - "total_tokens": "" - }, - "output": { - "generations": [ - [ - { - "generationInfo": { - "finish_reason": "stop" - }, - "message": { - "id": [ - "langchain_core", - "messages", - "AIMessage" - ], - "kwargs": { - "additional_kwargs": {}, - "content": "", - "id": null, - "invalid_tool_calls": [], - "response_metadata": { - "finish_reason": "stop", - "model_name": "gpt-4o-mini-2024-07-18", - "model_provider": "openai", - "system_fingerprint": "", - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - }, - "usage": { - "completion_tokens": "", - "completion_tokens_details": { - "accepted_prediction_tokens": "", - "audio_tokens": "", - "reasoning_tokens": "", - "rejected_prediction_tokens": "" - }, - "prompt_tokens": "", - "prompt_tokens_details": { - "audio_tokens": "", - "cached_tokens": "" - }, - "total_tokens": "" - } - }, - "tool_calls": [], - "type": "ai", - "usage_metadata": { - "input_token_details": { - "audio": "", - "cache_read": "" - }, - "input_tokens": "", - "output_token_details": { - "audio": "", - "reasoning": "" - }, - "output_tokens": "", - "total_tokens": "" - } - }, - "lc": 1, - "type": "constructor" - }, - "text": "" - } - ] - ], - "llmOutput": { - "tokenUsage": { - "completionTokens": "", - "promptTokens": "", - "totalTokens": "" - } - } - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 13, - "name": "ChatOpenAI", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-events.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-events.json deleted file mode 100644 index f23c2e800..000000000 --- a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-events.json +++ /dev/null @@ -1,204 +0,0 @@ -[ - { - "has_input": false, - "has_output": false, - "metadata": { - "scenario": "wrap-langchain-js-traces" - }, - "metric_keys": [], - "name": "langchain-wrapper-root", - "root_span_id": "", - "span_id": "", - "span_parents": [], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "invoke" - }, - "metric_keys": [], - "name": "langchain-invoke-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "total_tokens" - ], - "name": "ChatOpenAI", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "chain" - }, - "metric_keys": [], - "name": "langchain-chain-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": null, - "metric_keys": [], - "name": "RunnableSequence", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "task" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "stream" - }, - "metric_keys": [], - "name": "langchain-stream-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "time_to_first_token", - "total_tokens" - ], - "name": "ChatOpenAI", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool" - }, - "metric_keys": [], - "name": "langchain-tool-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "total_tokens" - ], - "name": "ChatOpenAI", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": false, - "has_output": false, - "metadata": { - "operation": "tool-result" - }, - "metric_keys": [], - "name": "langchain-tool-result-operation", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": null - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "total_tokens" - ], - "name": "ChatOpenAI", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - }, - { - "has_input": true, - "has_output": true, - "metadata": { - "model": "gpt-4o-mini-2024-07-18" - }, - "metric_keys": [ - "completion_tokens", - "prompt_cached_tokens", - "prompt_tokens", - "total_tokens" - ], - "name": "ChatOpenAI", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ], - "type": "llm" - } -] diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json new file mode 100644 index 000000000..095d9855d --- /dev/null +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.json @@ -0,0 +1,1560 @@ +{ + "span_tree": [ + { + "name": "langchain-wrapper-root", + "type": "task", + "children": [ + { + "name": "langchain-invoke-operation", + "children": [ + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "Reply with exactly OK.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "stop" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "OK.", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 2, + "promptTokens": 12, + "totalTokens": 14 + }, + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 12, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 14 + } + }, + "tool_calls": [], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 12, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 2, + "total_tokens": 14 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "OK." + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 2, + "promptTokens": 12, + "totalTokens": 14 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0 + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 24, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": {}, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 24, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "total_tokens": 14 + } + } + ], + "metadata": { + "operation": "invoke", + "testRunId": "" + } + }, + { + "name": "langchain-chain-operation", + "children": [ + { + "name": "RunnableSequence", + "type": "task", + "children": [ + { + "name": "ChatPromptTemplate", + "type": "task", + "children": [], + "input": { + "word": "PARIS" + }, + "output": { + "id": [ + "langchain_core", + "prompt_values", + "ChatPromptValue" + ], + "kwargs": { + "messages": [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "Reply with the single word PARIS and nothing else.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + }, + "lc": 1, + "type": "constructor" + }, + "metadata": { + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "metadata": {}, + "name": "ChatPromptTemplate", + "parent_run_id": "", + "run_id": "", + "run_type": "prompt", + "serialized": { + "id": [ + "langchain_core", + "prompts", + "chat", + "ChatPromptTemplate" + ], + "kwargs": { + "input_variables": [ + "word" + ], + "messages": [ + { + "id": [ + "langchain_core", + "prompts", + "chat", + "HumanMessagePromptTemplate" + ], + "kwargs": { + "prompt": { + "id": [ + "langchain_core", + "prompts", + "prompt", + "PromptTemplate" + ], + "kwargs": { + "input_variables": [ + "word" + ], + "template": "Reply with the single word {word} and nothing else.", + "template_format": "f-string" + }, + "lc": 1, + "type": "constructor" + } + }, + "lc": 1, + "type": "constructor" + } + ] + }, + "lc": 1, + "type": "constructor" + }, + "tags": [ + "seq:step:1" + ] + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "Reply with the single word PARIS and nothing else.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "stop" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "PARIS", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 18, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 20 + } + }, + "tool_calls": [], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 18, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 2, + "total_tokens": 20 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "PARIS" + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0 + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 32, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": {}, + "parent_run_id": "", + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [ + "seq:step:2" + ] + }, + "metrics": { + "completion_tokens": 2, + "prompt_cached_tokens": 0, + "prompt_tokens": 18, + "total_tokens": 20 + } + } + ], + "input": { + "word": "PARIS" + }, + "output": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "PARIS", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 2, + "promptTokens": 18, + "totalTokens": 20 + }, + "usage": { + "completion_tokens": 2, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 18, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 20 + } + }, + "tool_calls": [], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 18, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 2, + "total_tokens": 20 + } + }, + "lc": 1, + "type": "constructor" + }, + "metadata": { + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "metadata": {}, + "run_id": "", + "serialized": { + "id": [ + "langchain_core", + "runnables", + "RunnableSequence" + ], + "kwargs": { + "first": { + "id": [ + "langchain_core", + "prompts", + "chat", + "ChatPromptTemplate" + ], + "kwargs": { + "input_variables": [ + "word" + ], + "messages": [ + { + "id": [ + "langchain_core", + "prompts", + "chat", + "HumanMessagePromptTemplate" + ], + "kwargs": { + "prompt": { + "id": [ + "langchain_core", + "prompts", + "prompt", + "PromptTemplate" + ], + "kwargs": { + "input_variables": [ + "word" + ], + "template": "Reply with the single word {word} and nothing else.", + "template_format": "f-string" + }, + "lc": 1, + "type": "constructor" + } + }, + "lc": 1, + "type": "constructor" + } + ] + }, + "lc": 1, + "type": "constructor" + }, + "last": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + } + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + } + } + ], + "metadata": { + "operation": "chain", + "testRunId": "" + } + }, + { + "name": "langchain-stream-operation", + "children": [ + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "Count from 1 to 3 and include the words one two three.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "completion": 0, + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "prompt": 0, + "service_tier": "default", + "system_fingerprint": "" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessageChunk" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "One, two, three.", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "completion": 0, + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "prompt": 0, + "service_tier": "default", + "system_fingerprint": "", + "usage": { + "completion_tokens": 6, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 22, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 28 + } + }, + "tool_call_chunks": [], + "tool_calls": [], + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 22, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 6, + "total_tokens": 28 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "One, two, three." + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 6, + "promptTokens": 22, + "totalTokens": 28 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0 + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 32, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "stream": true, + "stream_options": { + "include_usage": true + }, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": {}, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 32, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "streaming": true, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + }, + "metrics": { + "completion_tokens": 6, + "prompt_cached_tokens": 0, + "prompt_tokens": 22, + "time_to_first_token": 0, + "total_tokens": 28 + } + } + ], + "metadata": { + "operation": "stream", + "testRunId": "" + } + }, + { + "name": "langchain-tool-operation", + "children": [ + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "tool_calls" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": { + "tool_calls": [ + { + "function": { + "arguments": "{\"location\":\"Paris, France\"}", + "name": "get_weather" + }, + "id": "", + "type": "function" + } + ] + }, + "content": "", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "tool_calls", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 16, + "promptTokens": 72, + "totalTokens": 88 + }, + "usage": { + "completion_tokens": 16, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 72, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 88 + } + }, + "tool_calls": [ + { + "args": { + "location": "Paris, France" + }, + "id": "", + "name": "get_weather", + "type": "tool_call" + } + ], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 72, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 16, + "total_tokens": 88 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "" + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 16, + "promptTokens": 72, + "totalTokens": 88 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "tools": [ + { + "function": { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 128, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": { + "tools": [ + { + "function": { + "description": "Get the current weather in a given location", + "name": "get_weather", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "location": { + "description": "The city and state or city and country", + "type": "string" + } + }, + "required": [ + "location" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + }, + "metrics": { + "completion_tokens": 16, + "prompt_cached_tokens": 0, + "prompt_tokens": 72, + "total_tokens": 88 + } + } + ], + "metadata": { + "operation": "tool", + "testRunId": "" + } + }, + { + "name": "langchain-tool-result-operation", + "children": [ + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "What is 127 multiplied by 49? Use the calculate tool.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "tool_calls" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": { + "tool_calls": [ + { + "function": { + "arguments": "{\"operation\":\"multiply\",\"a\":127,\"b\":49}", + "name": "calculate" + }, + "id": "", + "type": "function" + } + ] + }, + "content": "", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "tool_calls", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 21, + "promptTokens": 76, + "totalTokens": 97 + }, + "usage": { + "completion_tokens": 21, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 76, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 97 + } + }, + "tool_calls": [ + { + "args": { + "a": 127, + "b": 49, + "operation": "multiply" + }, + "id": "", + "name": "calculate", + "type": "tool_call" + } + ], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 76, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 21, + "total_tokens": 97 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "" + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 21, + "promptTokens": 76, + "totalTokens": 97 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 128, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": { + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + }, + "metrics": { + "completion_tokens": 21, + "prompt_cached_tokens": 0, + "prompt_tokens": 76, + "total_tokens": 97 + } + }, + { + "name": "ChatOpenAI", + "type": "llm", + "children": [], + "input": [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "What is 127 multiplied by 49? Use the calculate tool.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + }, + { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "", + "invalid_tool_calls": [], + "response_metadata": {}, + "tool_calls": [ + { + "args": { + "a": 127, + "b": 49, + "operation": "multiply" + }, + "id": "", + "name": "calculate", + "type": "tool_call" + } + ] + }, + "lc": 1, + "type": "constructor" + }, + { + "id": [ + "langchain_core", + "messages", + "ToolMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "6223", + "response_metadata": {}, + "tool_call_id": "call_V61pHdchBYqDmROrrAZ2d664" + }, + "lc": 1, + "type": "constructor" + } + ] + ], + "output": { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "stop" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "127 multiplied by 49 is 6223.", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 11, + "promptTokens": 106, + "totalTokens": 117 + }, + "usage": { + "completion_tokens": 11, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 106, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 117 + } + }, + "tool_calls": [], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 106, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 11, + "total_tokens": 117 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "127 multiplied by 49 is 6223." + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 11, + "promptTokens": 106, + "totalTokens": 117 + } + } + }, + "metadata": { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 128, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": { + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + }, + "metrics": { + "completion_tokens": 11, + "prompt_cached_tokens": 0, + "prompt_tokens": 106, + "total_tokens": 117 + } + } + ], + "metadata": { + "operation": "tool-result", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "wrap-langchain-js-traces", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt new file mode 100644 index 000000000..9304d284a --- /dev/null +++ b/e2e/scenarios/wrap-langchain-js-traces/__snapshots__/span-tree.txt @@ -0,0 +1,1499 @@ +span_tree: +└── langchain-wrapper-root [task] + metadata: { + "scenario": "wrap-langchain-js-traces", + "testRunId": "" + } + ├── langchain-invoke-operation + │ metadata: { + │ "operation": "invoke", + │ "testRunId": "" + │ } + │ └── ChatOpenAI [llm] + │ input: [ + │ [ + │ { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "HumanMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "Reply with exactly OK.", + │ "response_metadata": {} + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ ] + │ output: { + │ "generations": [ + │ [ + │ { + │ "generationInfo": { + │ "finish_reason": "stop" + │ }, + │ "message": { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "OK.", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "finish_reason": "stop", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "system_fingerprint": "", + │ "tokenUsage": { + │ "completionTokens": 2, + │ "promptTokens": 12, + │ "totalTokens": 14 + │ }, + │ "usage": { + │ "completion_tokens": 2, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 12, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 14 + │ } + │ }, + │ "tool_calls": [], + │ "type": "ai", + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 12, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 2, + │ "total_tokens": 14 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "text": "OK." + │ } + │ ] + │ ], + │ "llmOutput": { + │ "tokenUsage": { + │ "completionTokens": 2, + │ "promptTokens": 12, + │ "totalTokens": 14 + │ } + │ } + │ } + │ metadata: { + │ "batch_size": 1, + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "invocation_params": { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0 + │ }, + │ "metadata": { + │ "ls_integration": "langchain_chat_model", + │ "ls_max_tokens": 24, + │ "ls_model_name": "gpt-4o-mini-2024-07-18", + │ "ls_model_type": "chat", + │ "ls_provider": "openai", + │ "ls_temperature": 0, + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "versions": { + │ "@langchain/core": "1.1.35", + │ "@langchain/openai": "1.3.0" + │ } + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": {}, + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 24, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [] + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 12, + │ "total_tokens": 14 + │ } + ├── langchain-chain-operation + │ metadata: { + │ "operation": "chain", + │ "testRunId": "" + │ } + │ └── RunnableSequence [task] + │ input: { + │ "word": "PARIS" + │ } + │ output: { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "PARIS", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "finish_reason": "stop", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "system_fingerprint": "", + │ "tokenUsage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "completion_tokens": 2, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 18, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 20 + │ } + │ }, + │ "tool_calls": [], + │ "type": "ai", + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 18, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 2, + │ "total_tokens": 20 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ metadata: { + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "metadata": {}, + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain_core", + │ "runnables", + │ "RunnableSequence" + │ ], + │ "kwargs": { + │ "first": { + │ "id": [ + │ "langchain_core", + │ "prompts", + │ "chat", + │ "ChatPromptTemplate" + │ ], + │ "kwargs": { + │ "input_variables": [ + │ "word" + │ ], + │ "messages": [ + │ { + │ "id": [ + │ "langchain_core", + │ "prompts", + │ "chat", + │ "HumanMessagePromptTemplate" + │ ], + │ "kwargs": { + │ "prompt": { + │ "id": [ + │ "langchain_core", + │ "prompts", + │ "prompt", + │ "PromptTemplate" + │ ], + │ "kwargs": { + │ "input_variables": [ + │ "word" + │ ], + │ "template": "Reply with the single word {word} and nothing else.", + │ "template_format": "f-string" + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "last": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [] + │ } + │ ├── ChatPromptTemplate [task] + │ │ input: { + │ │ "word": "PARIS" + │ │ } + │ │ output: { + │ │ "id": [ + │ │ "langchain_core", + │ │ "prompt_values", + │ │ "ChatPromptValue" + │ │ ], + │ │ "kwargs": { + │ │ "messages": [ + │ │ { + │ │ "id": [ + │ │ "langchain_core", + │ │ "messages", + │ │ "HumanMessage" + │ │ ], + │ │ "kwargs": { + │ │ "additional_kwargs": {}, + │ │ "content": "Reply with the single word PARIS and nothing else.", + │ │ "response_metadata": {} + │ │ }, + │ │ "lc": 1, + │ │ "type": "constructor" + │ │ } + │ │ ] + │ │ }, + │ │ "lc": 1, + │ │ "type": "constructor" + │ │ } + │ │ metadata: { + │ │ "braintrust": { + │ │ "integration_name": "langchain-js", + │ │ "integration_version": "0.2.0", + │ │ "sdk_language": "javascript" + │ │ }, + │ │ "metadata": {}, + │ │ "name": "ChatPromptTemplate", + │ │ "parent_run_id": "", + │ │ "run_id": "", + │ │ "run_type": "prompt", + │ │ "serialized": { + │ │ "id": [ + │ │ "langchain_core", + │ │ "prompts", + │ │ "chat", + │ │ "ChatPromptTemplate" + │ │ ], + │ │ "kwargs": { + │ │ "input_variables": [ + │ │ "word" + │ │ ], + │ │ "messages": [ + │ │ { + │ │ "id": [ + │ │ "langchain_core", + │ │ "prompts", + │ │ "chat", + │ │ "HumanMessagePromptTemplate" + │ │ ], + │ │ "kwargs": { + │ │ "prompt": { + │ │ "id": [ + │ │ "langchain_core", + │ │ "prompts", + │ │ "prompt", + │ │ "PromptTemplate" + │ │ ], + │ │ "kwargs": { + │ │ "input_variables": [ + │ │ "word" + │ │ ], + │ │ "template": "Reply with the single word {word} and nothing else.", + │ │ "template_format": "f-string" + │ │ }, + │ │ "lc": 1, + │ │ "type": "constructor" + │ │ } + │ │ }, + │ │ "lc": 1, + │ │ "type": "constructor" + │ │ } + │ │ ] + │ │ }, + │ │ "lc": 1, + │ │ "type": "constructor" + │ │ }, + │ │ "tags": [ + │ │ "seq:step:1" + │ │ ] + │ │ } + │ └── ChatOpenAI [llm] + │ input: [ + │ [ + │ { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "HumanMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "Reply with the single word PARIS and nothing else.", + │ "response_metadata": {} + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ ] + │ output: { + │ "generations": [ + │ [ + │ { + │ "generationInfo": { + │ "finish_reason": "stop" + │ }, + │ "message": { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "PARIS", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "finish_reason": "stop", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "system_fingerprint": "", + │ "tokenUsage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ }, + │ "usage": { + │ "completion_tokens": 2, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 18, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 20 + │ } + │ }, + │ "tool_calls": [], + │ "type": "ai", + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 18, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 2, + │ "total_tokens": 20 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "text": "PARIS" + │ } + │ ] + │ ], + │ "llmOutput": { + │ "tokenUsage": { + │ "completionTokens": 2, + │ "promptTokens": 18, + │ "totalTokens": 20 + │ } + │ } + │ } + │ metadata: { + │ "batch_size": 1, + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "invocation_params": { + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0 + │ }, + │ "metadata": { + │ "ls_integration": "langchain_chat_model", + │ "ls_max_tokens": 32, + │ "ls_model_name": "gpt-4o-mini-2024-07-18", + │ "ls_model_type": "chat", + │ "ls_provider": "openai", + │ "ls_temperature": 0, + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "versions": { + │ "@langchain/core": "1.1.35", + │ "@langchain/openai": "1.3.0" + │ } + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": {}, + │ "parent_run_id": "", + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [ + │ "seq:step:2" + │ ] + │ } + │ metrics: { + │ "completion_tokens": 2, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 18, + │ "total_tokens": 20 + │ } + ├── langchain-stream-operation + │ metadata: { + │ "operation": "stream", + │ "testRunId": "" + │ } + │ └── ChatOpenAI [llm] + │ input: [ + │ [ + │ { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "HumanMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "Count from 1 to 3 and include the words one two three.", + │ "response_metadata": {} + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ ] + │ output: { + │ "generations": [ + │ [ + │ { + │ "generationInfo": { + │ "completion": 0, + │ "finish_reason": "stop", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "prompt": 0, + │ "service_tier": "default", + │ "system_fingerprint": "" + │ }, + │ "message": { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessageChunk" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "One, two, three.", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "completion": 0, + │ "finish_reason": "stop", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "prompt": 0, + │ "service_tier": "default", + │ "system_fingerprint": "", + │ "usage": { + │ "completion_tokens": 6, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 22, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 28 + │ } + │ }, + │ "tool_call_chunks": [], + │ "tool_calls": [], + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 22, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 6, + │ "total_tokens": 28 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "text": "One, two, three." + │ } + │ ] + │ ], + │ "llmOutput": { + │ "tokenUsage": { + │ "completionTokens": 6, + │ "promptTokens": 22, + │ "totalTokens": 28 + │ } + │ } + │ } + │ metadata: { + │ "batch_size": 1, + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "invocation_params": { + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0 + │ }, + │ "metadata": { + │ "ls_integration": "langchain_chat_model", + │ "ls_max_tokens": 32, + │ "ls_model_name": "gpt-4o-mini-2024-07-18", + │ "ls_model_type": "chat", + │ "ls_provider": "openai", + │ "ls_temperature": 0, + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": true, + │ "stream_options": { + │ "include_usage": true + │ }, + │ "temperature": 0, + │ "versions": { + │ "@langchain/core": "1.1.35", + │ "@langchain/openai": "1.3.0" + │ } + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": {}, + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 32, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "streaming": true, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [] + │ } + │ metrics: { + │ "completion_tokens": 6, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 22, + │ "time_to_first_token": 0, + │ "total_tokens": 28 + │ } + ├── langchain-tool-operation + │ metadata: { + │ "operation": "tool", + │ "testRunId": "" + │ } + │ └── ChatOpenAI [llm] + │ input: [ + │ [ + │ { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "HumanMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + │ "response_metadata": {} + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ ] + │ output: { + │ "generations": [ + │ [ + │ { + │ "generationInfo": { + │ "finish_reason": "tool_calls" + │ }, + │ "message": { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": { + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"location\":\"Paris, France\"}", + │ "name": "get_weather" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ "content": "", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "finish_reason": "tool_calls", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "system_fingerprint": "", + │ "tokenUsage": { + │ "completionTokens": 16, + │ "promptTokens": 72, + │ "totalTokens": 88 + │ }, + │ "usage": { + │ "completion_tokens": 16, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 72, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 88 + │ } + │ }, + │ "tool_calls": [ + │ { + │ "args": { + │ "location": "Paris, France" + │ }, + │ "id": "", + │ "name": "get_weather", + │ "type": "tool_call" + │ } + │ ], + │ "type": "ai", + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 72, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 16, + │ "total_tokens": 88 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "text": "" + │ } + │ ] + │ ], + │ "llmOutput": { + │ "tokenUsage": { + │ "completionTokens": 16, + │ "promptTokens": 72, + │ "totalTokens": 88 + │ } + │ } + │ } + │ metadata: { + │ "batch_size": 1, + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "invocation_params": { + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the current weather in a given location", + │ "name": "get_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ }, + │ "metadata": { + │ "ls_integration": "langchain_chat_model", + │ "ls_max_tokens": 128, + │ "ls_model_name": "gpt-4o-mini-2024-07-18", + │ "ls_model_type": "chat", + │ "ls_provider": "openai", + │ "ls_temperature": 0, + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "versions": { + │ "@langchain/core": "1.1.35", + │ "@langchain/openai": "1.3.0" + │ } + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": { + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the current weather in a given location", + │ "name": "get_weather", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "location": { + │ "description": "The city and state or city and country", + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "location" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ }, + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [] + │ } + │ metrics: { + │ "completion_tokens": 16, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 72, + │ "total_tokens": 88 + │ } + └── langchain-tool-result-operation + metadata: { + "operation": "tool-result", + "testRunId": "" + } + ├── ChatOpenAI [llm] + │ input: [ + │ [ + │ { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "HumanMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": {}, + │ "content": "What is 127 multiplied by 49? Use the calculate tool.", + │ "response_metadata": {} + │ }, + │ "lc": 1, + │ "type": "constructor" + │ } + │ ] + │ ] + │ output: { + │ "generations": [ + │ [ + │ { + │ "generationInfo": { + │ "finish_reason": "tool_calls" + │ }, + │ "message": { + │ "id": [ + │ "langchain_core", + │ "messages", + │ "AIMessage" + │ ], + │ "kwargs": { + │ "additional_kwargs": { + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"operation\":\"multiply\",\"a\":127,\"b\":49}", + │ "name": "calculate" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ "content": "", + │ "id": "", + │ "invalid_tool_calls": [], + │ "response_metadata": { + │ "finish_reason": "tool_calls", + │ "model_name": "gpt-4o-mini-2024-07-18", + │ "model_provider": "openai", + │ "system_fingerprint": "", + │ "tokenUsage": { + │ "completionTokens": 21, + │ "promptTokens": 76, + │ "totalTokens": 97 + │ }, + │ "usage": { + │ "completion_tokens": 21, + │ "completion_tokens_details": { + │ "accepted_prediction_tokens": 0, + │ "audio_tokens": 0, + │ "reasoning_tokens": 0, + │ "rejected_prediction_tokens": 0 + │ }, + │ "prompt_tokens": 76, + │ "prompt_tokens_details": { + │ "audio_tokens": 0, + │ "cached_tokens": 0 + │ }, + │ "total_tokens": 97 + │ } + │ }, + │ "tool_calls": [ + │ { + │ "args": { + │ "a": 127, + │ "b": 49, + │ "operation": "multiply" + │ }, + │ "id": "", + │ "name": "calculate", + │ "type": "tool_call" + │ } + │ ], + │ "type": "ai", + │ "usage_metadata": { + │ "input_token_details": { + │ "audio": 0, + │ "cache_read": 0 + │ }, + │ "input_tokens": 76, + │ "output_token_details": { + │ "audio": 0, + │ "reasoning": 0 + │ }, + │ "output_tokens": 21, + │ "total_tokens": 97 + │ } + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "text": "" + │ } + │ ] + │ ], + │ "llmOutput": { + │ "tokenUsage": { + │ "completionTokens": 21, + │ "promptTokens": 76, + │ "totalTokens": 97 + │ } + │ } + │ } + │ metadata: { + │ "batch_size": 1, + │ "braintrust": { + │ "integration_name": "langchain-js", + │ "integration_version": "0.2.0", + │ "sdk_language": "javascript" + │ }, + │ "invocation_params": { + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "tools": [ + │ { + │ "function": { + │ "description": "Perform a mathematical calculation", + │ "name": "calculate", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "a": { + │ "type": "number" + │ }, + │ "b": { + │ "type": "number" + │ }, + │ "operation": { + │ "enum": [ + │ "add", + │ "subtract", + │ "multiply", + │ "divide" + │ ], + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "operation", + │ "a", + │ "b" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ }, + │ "metadata": { + │ "ls_integration": "langchain_chat_model", + │ "ls_max_tokens": 128, + │ "ls_model_name": "gpt-4o-mini-2024-07-18", + │ "ls_model_type": "chat", + │ "ls_provider": "openai", + │ "ls_temperature": 0, + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "stream": false, + │ "temperature": 0, + │ "versions": { + │ "@langchain/core": "1.1.35", + │ "@langchain/openai": "1.3.0" + │ } + │ }, + │ "model": "gpt-4o-mini-2024-07-18", + │ "options": { + │ "tools": [ + │ { + │ "function": { + │ "description": "Perform a mathematical calculation", + │ "name": "calculate", + │ "parameters": { + │ "$schema": "http://json-schema.org/draft-07/schema#", + │ "additionalProperties": false, + │ "properties": { + │ "a": { + │ "type": "number" + │ }, + │ "b": { + │ "type": "number" + │ }, + │ "operation": { + │ "enum": [ + │ "add", + │ "subtract", + │ "multiply", + │ "divide" + │ ], + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "operation", + │ "a", + │ "b" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ }, + │ "run_id": "", + │ "serialized": { + │ "id": [ + │ "langchain", + │ "chat_models", + │ "openai", + │ "ChatOpenAI" + │ ], + │ "kwargs": { + │ "max_tokens": 128, + │ "model": "gpt-4o-mini-2024-07-18", + │ "openai_api_key": { + │ "id": [ + │ "OPENAI_API_KEY" + │ ], + │ "lc": 1, + │ "type": "secret" + │ }, + │ "temperature": 0 + │ }, + │ "lc": 1, + │ "type": "constructor" + │ }, + │ "tags": [] + │ } + │ metrics: { + │ "completion_tokens": 21, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 76, + │ "total_tokens": 97 + │ } + └── ChatOpenAI [llm] + input: [ + [ + { + "id": [ + "langchain_core", + "messages", + "HumanMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "What is 127 multiplied by 49? Use the calculate tool.", + "response_metadata": {} + }, + "lc": 1, + "type": "constructor" + }, + { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "", + "invalid_tool_calls": [], + "response_metadata": {}, + "tool_calls": [ + { + "args": { + "a": 127, + "b": 49, + "operation": "multiply" + }, + "id": "", + "name": "calculate", + "type": "tool_call" + } + ] + }, + "lc": 1, + "type": "constructor" + }, + { + "id": [ + "langchain_core", + "messages", + "ToolMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "6223", + "response_metadata": {}, + "tool_call_id": "call_V61pHdchBYqDmROrrAZ2d664" + }, + "lc": 1, + "type": "constructor" + } + ] + ] + output: { + "generations": [ + [ + { + "generationInfo": { + "finish_reason": "stop" + }, + "message": { + "id": [ + "langchain_core", + "messages", + "AIMessage" + ], + "kwargs": { + "additional_kwargs": {}, + "content": "127 multiplied by 49 is 6223.", + "id": "", + "invalid_tool_calls": [], + "response_metadata": { + "finish_reason": "stop", + "model_name": "gpt-4o-mini-2024-07-18", + "model_provider": "openai", + "system_fingerprint": "", + "tokenUsage": { + "completionTokens": 11, + "promptTokens": 106, + "totalTokens": 117 + }, + "usage": { + "completion_tokens": 11, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens": 106, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + }, + "total_tokens": 117 + } + }, + "tool_calls": [], + "type": "ai", + "usage_metadata": { + "input_token_details": { + "audio": 0, + "cache_read": 0 + }, + "input_tokens": 106, + "output_token_details": { + "audio": 0, + "reasoning": 0 + }, + "output_tokens": 11, + "total_tokens": 117 + } + }, + "lc": 1, + "type": "constructor" + }, + "text": "127 multiplied by 49 is 6223." + } + ] + ], + "llmOutput": { + "tokenUsage": { + "completionTokens": 11, + "promptTokens": 106, + "totalTokens": 117 + } + } + } + metadata: { + "batch_size": 1, + "braintrust": { + "integration_name": "langchain-js", + "integration_version": "0.2.0", + "sdk_language": "javascript" + }, + "invocation_params": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_max_tokens": 128, + "ls_model_name": "gpt-4o-mini-2024-07-18", + "ls_model_type": "chat", + "ls_provider": "openai", + "ls_temperature": 0, + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "stream": false, + "temperature": 0, + "versions": { + "@langchain/core": "1.1.35", + "@langchain/openai": "1.3.0" + } + }, + "model": "gpt-4o-mini-2024-07-18", + "options": { + "tools": [ + { + "function": { + "description": "Perform a mathematical calculation", + "name": "calculate", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + }, + "operation": { + "enum": [ + "add", + "subtract", + "multiply", + "divide" + ], + "type": "string" + } + }, + "required": [ + "operation", + "a", + "b" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "run_id": "", + "serialized": { + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI" + ], + "kwargs": { + "max_tokens": 128, + "model": "gpt-4o-mini-2024-07-18", + "openai_api_key": { + "id": [ + "OPENAI_API_KEY" + ], + "lc": 1, + "type": "secret" + }, + "temperature": 0 + }, + "lc": 1, + "type": "constructor" + }, + "tags": [] + } + metrics: { + "completion_tokens": 11, + "prompt_cached_tokens": 0, + "prompt_tokens": 106, + "total_tokens": 117 + } diff --git a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts index 2f058d7b2..729049c40 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/assertions.ts +++ b/e2e/scenarios/wrap-langchain-js-traces/assertions.ts @@ -1,231 +1,6 @@ import { expect } from "vitest"; -import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; -import type { - CapturedLogEvent, - CapturedLogPayload, -} from "../../helpers/mock-braintrust-server"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; -import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "../../helpers/wrapper-contract"; - -/** - * Normalizes LangChain payload rows to make snapshots deterministic by - * replacing non-deterministic LLM output content and token counts with - * stable placeholders. - */ -function normalizeLangchainPayloads(payloadRows: unknown[]): unknown[] { - return payloadRows.map((payload) => { - if (!payload || typeof payload !== "object") { - return payload; - } - - const row = structuredClone(payload) as Record; - - if (row.metrics && typeof row.metrics === "object") { - const metrics = row.metrics as Record; - for (const key of Object.keys(metrics)) { - if (key.includes("token") && typeof metrics[key] === "number") { - metrics[key] = ""; - } - if (key === "time_to_first_token") { - metrics[key] = 0; - } - } - } - - if (row.output && typeof row.output === "object") { - normalizeOutputObject(row.output as Record); - } - - normalizeToolCallIds(row); - normalizeLangchainGeneratedIds(row); - normalizeLangchainVersions(row); - - return row; - }); -} - -function normalizeOutputObject(obj: Record): void { - if (obj.llmOutput && typeof obj.llmOutput === "object") { - const llmOutput = obj.llmOutput as Record; - if (llmOutput.tokenUsage && typeof llmOutput.tokenUsage === "object") { - const tokenUsage = llmOutput.tokenUsage as Record; - for (const key of Object.keys(tokenUsage)) { - tokenUsage[key] = ""; - } - } - } - - if (Array.isArray(obj.generations)) { - for (const batch of obj.generations) { - if (!Array.isArray(batch)) continue; - for (const generation of batch) { - if (!generation || typeof generation !== "object") continue; - const normalizedGeneration = generation as Record; - if (typeof normalizedGeneration.text === "string") { - normalizedGeneration.text = ""; - } - if ( - normalizedGeneration.message && - typeof normalizedGeneration.message === "object" - ) { - normalizeMessageObject( - normalizedGeneration.message as Record, - ); - } - } - } - } -} - -function normalizeMessageObject(message: Record): void { - const kwargs = message.kwargs as Record | undefined; - if (!kwargs) return; - - if (typeof kwargs.id === "string") { - kwargs.id = ""; - } - - if (typeof kwargs.content === "string" && kwargs.content !== "") { - kwargs.content = ""; - } - - normalizeTokenCounts(kwargs.usage_metadata); - if ( - kwargs.response_metadata && - typeof kwargs.response_metadata === "object" - ) { - const responseMetadata = kwargs.response_metadata as Record< - string, - unknown - >; - normalizeTokenCounts(responseMetadata.tokenUsage); - normalizeTokenCounts(responseMetadata.usage); - } -} - -function normalizeLangchainGeneratedIds( - obj: unknown, - path: string[] = [], -): void { - if (!obj || typeof obj !== "object") return; - - if (Array.isArray(obj)) { - for (const item of obj) { - normalizeLangchainGeneratedIds(item, path); - } - return; - } - - const record = obj as Record; - for (const [key, value] of Object.entries(record)) { - if ( - (key === "run_id" || key === "parent_run_id") && - typeof value === "string" - ) { - record[key] = ``; - continue; - } - - if (key === "id" && typeof value === "string" && path.length > 0) { - record[key] = null; - continue; - } - - if (typeof value === "object" && value !== null) { - normalizeLangchainGeneratedIds(value, [...path, key]); - } - } -} - -function normalizeTokenCounts(obj: unknown): void { - if (!obj || typeof obj !== "object") return; - const record = obj as Record; - for (const [key, value] of Object.entries(record)) { - if (typeof value === "number") { - record[key] = ""; - } else if (typeof value === "object" && value !== null) { - normalizeTokenCounts(value); - } - } -} - -function normalizeToolCallIds(obj: unknown): void { - if (!obj || typeof obj !== "object") return; - - if (Array.isArray(obj)) { - for (const item of obj) { - normalizeToolCallIds(item); - } - return; - } - - const record = obj as Record; - for (const [key, value] of Object.entries(record)) { - if ( - (key === "tool_call_id" || key === "id") && - typeof value === "string" && - value.startsWith("call_") - ) { - record[key] = ""; - } else if (typeof value === "object" && value !== null) { - normalizeToolCallIds(value); - } - } -} - -// Fields that older @langchain/openai included in the ls_* metadata block but -// newer versions removed. Normalize them out so the snapshot is stable across -// both locked and canary (latest) langchain versions. -const LANGCHAIN_LS_VOLATILE_KEYS = new Set([ - "max_tokens", - "model", - "stream", - "stream_options", - "temperature", -]); - -function normalizeLangchainVersions(obj: unknown): void { - if (!obj || typeof obj !== "object") return; - - if (Array.isArray(obj)) { - for (const item of obj) { - normalizeLangchainVersions(item); - } - return; - } - - const record = obj as Record; - if ( - record.versions && - typeof record.versions === "object" && - !Array.isArray(record.versions) - ) { - const versions = record.versions as Record; - for (const key of Object.keys(versions)) { - if (key.startsWith("@langchain/") && typeof versions[key] === "string") { - versions[key] = ""; - } - } - } - - // If this object is the ls_* metadata block (identified by the presence of - // any `ls_` key), remove volatile keys that newer langchain drops. - const hasLsKey = Object.keys(record).some((k) => k.startsWith("ls_")); - if (hasLsKey) { - for (const key of LANGCHAIN_LS_VOLATILE_KEYS) { - delete record[key]; - } - } - - for (const value of Object.values(record)) { - if (typeof value === "object" && value !== null) { - normalizeLangchainVersions(value); - } - } -} function findNamedChildSpan( capturedEvents: CapturedLogEvent[], @@ -244,10 +19,9 @@ function findNamedChildSpan( export function assertLangchainTraces(options: { capturedEvents: CapturedLogEvent[]; - payloads: CapturedLogPayload[]; rootName: string; scenarioName: string; -}): { payloadSummary: Json; spanSummary: Json } { +}): CapturedLogEvent[] { const root = findLatestSpan(options.capturedEvents, options.rootName); const invokeOperation = findLatestSpan( options.capturedEvents, @@ -334,28 +108,5 @@ export function assertLangchainTraces(options: { ); expect(toolResultSpans.length).toBeGreaterThanOrEqual(2); - return { - spanSummary: normalizeForSnapshot( - [ - root, - invokeOperation, - invokeSpan, - chainOperation, - ...chainChildren, - streamOperation, - streamSpan, - toolOperation, - toolSpan, - toolResultOperation, - ...toolResultSpans, - ].map((event) => - summarizeWrapperContract(event!, ["model", "operation", "scenario"]), - ) as Json, - ), - payloadSummary: normalizeForSnapshot( - normalizeLangchainPayloads( - payloadRowsForRootSpan(options.payloads, root?.span.id), - ) as Json, - ), - }; + return options.capturedEvents; } diff --git a/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts b/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts index 5aa071986..3ecb4df78 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts +++ b/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts @@ -1,14 +1,11 @@ import { expect, test } from "vitest"; -import { - formatJsonFileSnapshot, - matchFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; import { prepareScenarioDir, resolveScenarioDir, withScenarioHarness, } from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; import { assertLangchainTraces } from "./assertions"; @@ -25,7 +22,7 @@ test( timeout: TIMEOUT_MS, }, async () => { - await withScenarioHarness(async ({ events, payloads, runScenarioDir }) => { + await withScenarioHarness(async ({ events, runScenarioDir }) => { await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS, @@ -35,20 +32,15 @@ test( }, }); - const summaries = assertLangchainTraces({ + const spanTree = assertLangchainTraces({ capturedEvents: events(), - payloads: payloads(), rootName: "langchain-wrapper-root", scenarioName: "wrap-langchain-js-traces", }); - await matchFileSnapshot( - formatJsonFileSnapshot(summaries.spanSummary), - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await matchFileSnapshot( - formatJsonFileSnapshot(summaries.payloadSummary), - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + await matchSpanTreeSnapshot( + spanTree, + resolveFileSnapshotPath(import.meta.url, "span-tree.json"), ); }); }, diff --git a/e2e/scripts/run-e2e-tests.mjs b/e2e/scripts/run-e2e-tests.mjs new file mode 100644 index 000000000..2d7108f25 --- /dev/null +++ b/e2e/scripts/run-e2e-tests.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); +const E2E_DIR = path.resolve(SCRIPT_DIR, ".."); +const VITEST_COMMAND = path.join( + E2E_DIR, + "node_modules", + ".bin", + process.platform === "win32" ? "vitest.cmd" : "vitest", +); +const DEFAULT_OPENAI_CODEX_E2E_MODEL = "gpt-5.1-codex-mini"; + +const rawArgs = process.argv.slice(2).filter((arg) => arg !== "--"); +const updateSnapshots = rawArgs.includes("--update"); +const scenarioArgs = rawArgs.filter((arg) => arg !== "--update"); +const vitestArgs = [ + "run", + "--run", + ...scenarioArgs.map((arg) => scenarioPathArg(arg)), + ...(updateSnapshots ? ["--update"] : []), +]; +const result = await runProcess(VITEST_COMMAND, vitestArgs, { + cwd: E2E_DIR, + env: replayEnv(), +}); + +if (result.signal) { + console.error(`[e2e] Vitest exited after receiving ${result.signal}.`); + process.exit(1); +} +process.exit(result.code ?? 1); + +function scenarioPathArg(arg) { + if (!/^[a-z0-9][a-z0-9-]*$/.test(arg)) { + return arg; + } + + const scenarioPath = path.join(E2E_DIR, "scenarios", arg, "scenario.test.ts"); + return existsSync(scenarioPath) ? `scenarios/${arg}/scenario.test.ts` : arg; +} + +function replayEnv() { + const env = { ...process.env }; + env.BRAINTRUST_E2E_CASSETTE_MODE = "replay"; + env.OPENAI_CODEX_E2E_MODEL = DEFAULT_OPENAI_CODEX_E2E_MODEL; + return env; +} + +async function runProcess(command, args, options) { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: "inherit", + }); + child.on("error", reject); + child.on("close", (code, signal) => resolve({ code, signal })); + }); +} diff --git a/package.json b/package.json index 6386d33f5..0ada1ad93 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "knip": "knip --config knip.jsonc --no-config-hints", "test": "turbo run test --filter=\"!@braintrust/otel\"", "test:e2e": "turbo run test:e2e", + "test:e2e:update": "turbo run test:e2e:update", "test:e2e:record": "node e2e/scripts/run-record-tests.mjs --braintrust-build-deps", "test:e2e:canary": "turbo run test:e2e:canary", "changeset": "changeset", diff --git a/turbo.json b/turbo.json index a8aee0764..ae460ea25 100644 --- a/turbo.json +++ b/turbo.json @@ -1,17 +1,5 @@ { "$schema": "https://turbo.build/schema.json", - "globalPassThroughEnv": [ - "OPENAI_API_KEY", - "OPENAI_BASE_URL", - "ANTHROPIC_API_KEY", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], "tasks": { "build": { "dependsOn": ["^build"], @@ -46,22 +34,11 @@ }, "test:e2e": { "cache": false, - "env": [ - "ANTHROPIC_API_KEY", - "BRAINTRUST_API_KEY", - "BRAINTRUST_E2E_CASSETTE_MODE", - "BRAINTRUST_E2E_PROJECT_NAME", - "BRAINTRUST_E2E_RUN_CONTEXT_DIR", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENAI_API_KEY", - "OPENAI_BASE_URL", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], + "dependsOn": ["^build"], + "outputs": [] + }, + "test:e2e:update": { + "cache": false, "dependsOn": ["^build"], "outputs": [] },