diff --git a/plugins/tracing/dist/index.mjs b/plugins/tracing/dist/index.mjs index cdc80c5..f96eb74 100644 --- a/plugins/tracing/dist/index.mjs +++ b/plugins/tracing/dist/index.mjs @@ -46981,15 +46981,28 @@ async function seededTraceParent(config$1, sessionMeta, turnNumber) { return; } } +function isTokenCount(value) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} +/** Send Codex's inclusive counts using Langfuse's strict OpenAI usage schema. */ function toUsageDetails(usage) { if (!usage) return void 0; - const details = {}; - if (typeof usage.input_tokens === "number") details.input = usage.input_tokens; - if (typeof usage.output_tokens === "number") details.output = usage.output_tokens; - if (typeof usage.total_tokens === "number") details.total = usage.total_tokens; - if (typeof usage.cached_input_tokens === "number") details.cache_read_input_tokens = usage.cached_input_tokens; - if (typeof usage.reasoning_output_tokens === "number") details.reasoning_tokens = usage.reasoning_output_tokens; - return Object.keys(details).length > 0 ? details : void 0; + const { input_tokens: input, output_tokens: output, total_tokens: total, cached_input_tokens: cached$1, reasoning_output_tokens: reasoning } = usage; + if (!isTokenCount(input) || !isTokenCount(output) || !isTokenCount(total) || total !== input + output) { + debugLog("dropping usage: missing or inconsistent token counts", usage); + return; + } + if (cached$1 !== void 0 && (!isTokenCount(cached$1) || cached$1 > input) || reasoning !== void 0 && (!isTokenCount(reasoning) || reasoning > output)) { + debugLog("dropping usage: implausible cached/reasoning details", usage); + return; + } + return { + prompt_tokens: input, + completion_tokens: output, + total_tokens: total, + ...cached$1 !== void 0 ? { prompt_tokens_details: { cached_tokens: cached$1 } } : {}, + ...reasoning !== void 0 ? { completion_tokens_details: { reasoning_tokens: reasoning } } : {} + }; } /** Build a clip() that truncates long strings to `maxChars`. */ function makeClip(maxChars) { diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index 3f5e2d0..24b4f0e 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -6,6 +6,7 @@ import { createTraceId, propagateAttributes, startObservation, + type LangfuseGenerationAttributes, type LangfuseObservation, } from "@langfuse/tracing"; import { TraceFlags, type SpanContext } from "@opentelemetry/api"; @@ -109,19 +110,51 @@ async function seededTraceParent( } } -function toUsageDetails(usage: TokenUsage | undefined): Record | undefined { +function isTokenCount(value: number | undefined): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +/** Send Codex's inclusive counts using Langfuse's strict OpenAI usage schema. */ +function toUsageDetails( + usage: TokenUsage | undefined, +): LangfuseGenerationAttributes["usageDetails"] { if (!usage) return undefined; - const details: Record = {}; - if (typeof usage.input_tokens === "number") details.input = usage.input_tokens; - if (typeof usage.output_tokens === "number") details.output = usage.output_tokens; - if (typeof usage.total_tokens === "number") details.total = usage.total_tokens; - if (typeof usage.cached_input_tokens === "number") { - details.cache_read_input_tokens = usage.cached_input_tokens; + const { + input_tokens: input, + output_tokens: output, + total_tokens: total, + cached_input_tokens: cached, + reasoning_output_tokens: reasoning, + } = usage; + + if ( + !isTokenCount(input) || + !isTokenCount(output) || + !isTokenCount(total) || + total !== input + output + ) { + debugLog("dropping usage: missing or inconsistent token counts", usage); + return undefined; } - if (typeof usage.reasoning_output_tokens === "number") { - details.reasoning_tokens = usage.reasoning_output_tokens; + if ( + (cached !== undefined && (!isTokenCount(cached) || cached > input)) || + (reasoning !== undefined && (!isTokenCount(reasoning) || reasoning > output)) + ) { + debugLog("dropping usage: implausible cached/reasoning details", usage); + return undefined; } - return Object.keys(details).length > 0 ? details : undefined; + + // The runtime supports this documented shape, but the SDK type still + // exposes only its legacy camelCase usage interface. + return { + prompt_tokens: input, + completion_tokens: output, + total_tokens: total, + ...(cached !== undefined ? { prompt_tokens_details: { cached_tokens: cached } } : {}), + ...(reasoning !== undefined + ? { completion_tokens_details: { reasoning_tokens: reasoning } } + : {}), + } as unknown as LangfuseGenerationAttributes["usageDetails"]; } type Clip = { diff --git a/plugins/tracing/src/types.ts b/plugins/tracing/src/types.ts index 5f1de5a..36abc2c 100644 --- a/plugins/tracing/src/types.ts +++ b/plugins/tracing/src/types.ts @@ -110,6 +110,7 @@ export type TurnContextPayload = { [key: string]: unknown; }; +/** Codex reports top-level input/output counts inclusive of their details. */ export type TokenUsage = { input_tokens?: number; output_tokens?: number; diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index e3c0c95..ca9890a 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -85,19 +85,38 @@ describe("convertRollout", () => { // Two generations, both children of the root, named "LLM" (the model name // lives in the model attribute, not the observation name). - const generations = spans.filter((s) => obsType(s) === "generation"); + const generations = spans + .filter((s) => obsType(s) === "generation") + .sort((a, b) => startMs(a) - startMs(b)); expect(generations).toHaveLength(2); for (const gen of generations) { expect(gen.name).toBe("LLM"); expect(parentId(gen)).toBe(root!.spanContext().spanId); expect(attr(gen, "langfuse.observation.model.name")).toBe("gpt-5.4"); } - // First generation carries token usage. - const usage = generations - .map((g) => attr(g, "langfuse.observation.usage_details")) - .find((u) => u.includes("120")); - expect(usage, "expected usage details with 120 total tokens").toBeTruthy(); - + // Usage is sent in Langfuse's OpenAI-compatible shape. Langfuse then + // normalizes the inclusive parent counts and nested detail counts. + const usages = generations.map((generation) => { + const usage = attr(generation, "langfuse.observation.usage_details"); + expect(usage, "expected generation usage details").not.toBe(""); + return JSON.parse(usage); + }); + expect(usages).toEqual([ + { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 5 }, + }, + { + prompt_tokens: 150, + completion_tokens: 30, + total_tokens: 180, + prompt_tokens_details: { cached_tokens: 50 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + ]); // One tool span, nested under a generation, with the captured command output. const tools = spans.filter((s) => obsType(s) === "tool"); expect(tools).toHaveLength(1);