From b0c02271b6c0eb2b13297429f2eea4112d1edd3f Mon Sep 17 00:00:00 2001 From: Valery Meleshkin Date: Wed, 5 Aug 2026 13:57:00 +0200 Subject: [PATCH 1/3] Normalize tracing usage to Langfuse OpenAI schema --- plugins/tracing/dist/index.mjs | 21 ++++++++---- plugins/tracing/src/trace.ts | 51 ++++++++++++++++++++++++------ plugins/tracing/src/types.ts | 1 + plugins/tracing/test/trace.test.ts | 27 ++++++++++++---- 4 files changed, 77 insertions(+), 23 deletions(-) diff --git a/plugins/tracing/dist/index.mjs b/plugins/tracing/dist/index.mjs index cdc80c5..d74aa5a 100644 --- a/plugins/tracing/dist/index.mjs +++ b/plugins/tracing/dist/index.mjs @@ -46981,15 +46981,22 @@ async function seededTraceParent(config$1, sessionMeta, turnNumber) { return; } } +function isTokenCount(value) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} +/** Send Codex's inclusive counts using Langfuse's strict OpenAI usage schema. */ function toUsageDetails(usage) { if (!usage) return void 0; - const details = {}; - if (typeof usage.input_tokens === "number") details.input = usage.input_tokens; - if (typeof usage.output_tokens === "number") details.output = usage.output_tokens; - if (typeof usage.total_tokens === "number") details.total = usage.total_tokens; - if (typeof usage.cached_input_tokens === "number") details.cache_read_input_tokens = usage.cached_input_tokens; - if (typeof usage.reasoning_output_tokens === "number") details.reasoning_tokens = usage.reasoning_output_tokens; - return Object.keys(details).length > 0 ? details : void 0; + const { input_tokens: input, output_tokens: output, total_tokens: total, cached_input_tokens: cached$1, reasoning_output_tokens: reasoning } = usage; + if (!isTokenCount(input) || !isTokenCount(output) || !isTokenCount(total) || total !== input + output) return; + if (cached$1 !== void 0 && (!isTokenCount(cached$1) || cached$1 > input) || reasoning !== void 0 && (!isTokenCount(reasoning) || reasoning > output)) return; + return { + prompt_tokens: input, + completion_tokens: output, + total_tokens: total, + ...cached$1 !== void 0 ? { prompt_tokens_details: { cached_tokens: cached$1 } } : {}, + ...reasoning !== void 0 ? { completion_tokens_details: { reasoning_tokens: reasoning } } : {} + }; } /** Build a clip() that truncates long strings to `maxChars`. */ function makeClip(maxChars) { diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index 3f5e2d0..ffb68c7 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -6,6 +6,7 @@ import { createTraceId, propagateAttributes, startObservation, + type LangfuseGenerationAttributes, type LangfuseObservation, } from "@langfuse/tracing"; import { TraceFlags, type SpanContext } from "@opentelemetry/api"; @@ -109,19 +110,49 @@ async function seededTraceParent( } } -function toUsageDetails(usage: TokenUsage | undefined): Record | undefined { +function isTokenCount(value: number | undefined): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +/** Send Codex's inclusive counts using Langfuse's strict OpenAI usage schema. */ +function toUsageDetails( + usage: TokenUsage | undefined, +): LangfuseGenerationAttributes["usageDetails"] { if (!usage) return undefined; - const details: Record = {}; - if (typeof usage.input_tokens === "number") details.input = usage.input_tokens; - if (typeof usage.output_tokens === "number") details.output = usage.output_tokens; - if (typeof usage.total_tokens === "number") details.total = usage.total_tokens; - if (typeof usage.cached_input_tokens === "number") { - details.cache_read_input_tokens = usage.cached_input_tokens; + const { + input_tokens: input, + output_tokens: output, + total_tokens: total, + cached_input_tokens: cached, + reasoning_output_tokens: reasoning, + } = usage; + + if ( + !isTokenCount(input) || + !isTokenCount(output) || + !isTokenCount(total) || + total !== input + output + ) { + return undefined; } - if (typeof usage.reasoning_output_tokens === "number") { - details.reasoning_tokens = usage.reasoning_output_tokens; + if ( + (cached !== undefined && (!isTokenCount(cached) || cached > input)) || + (reasoning !== undefined && (!isTokenCount(reasoning) || reasoning > output)) + ) { + return undefined; } - return Object.keys(details).length > 0 ? details : undefined; + + // The runtime supports this documented shape, but the SDK type still + // exposes only its legacy camelCase usage interface. + return { + prompt_tokens: input, + completion_tokens: output, + total_tokens: total, + ...(cached !== undefined ? { prompt_tokens_details: { cached_tokens: cached } } : {}), + ...(reasoning !== undefined + ? { completion_tokens_details: { reasoning_tokens: reasoning } } + : {}), + } as unknown as LangfuseGenerationAttributes["usageDetails"]; } type Clip = { diff --git a/plugins/tracing/src/types.ts b/plugins/tracing/src/types.ts index 5f1de5a..36abc2c 100644 --- a/plugins/tracing/src/types.ts +++ b/plugins/tracing/src/types.ts @@ -110,6 +110,7 @@ export type TurnContextPayload = { [key: string]: unknown; }; +/** Codex reports top-level input/output counts inclusive of their details. */ export type TokenUsage = { input_tokens?: number; output_tokens?: number; diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index e3c0c95..6f0731f 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -92,12 +92,27 @@ describe("convertRollout", () => { expect(parentId(gen)).toBe(root!.spanContext().spanId); expect(attr(gen, "langfuse.observation.model.name")).toBe("gpt-5.4"); } - // First generation carries token usage. - const usage = generations - .map((g) => attr(g, "langfuse.observation.usage_details")) - .find((u) => u.includes("120")); - expect(usage, "expected usage details with 120 total tokens").toBeTruthy(); - + // Usage is sent in Langfuse's OpenAI-compatible shape. Langfuse then + // normalizes the inclusive parent counts and nested detail counts. + const usages = generations.map((g) => + JSON.parse(attr(g, "langfuse.observation.usage_details")), + ); + expect(usages).toEqual([ + { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 5 }, + }, + { + prompt_tokens: 150, + completion_tokens: 30, + total_tokens: 180, + prompt_tokens_details: { cached_tokens: 50 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + ]); // One tool span, nested under a generation, with the captured command output. const tools = spans.filter((s) => obsType(s) === "tool"); expect(tools).toHaveLength(1); From 4333c3462d7fd0d58cb0bbdf40d4ade98851cb3e Mon Sep 17 00:00:00 2001 From: Valery Meleshkin Date: Wed, 5 Aug 2026 14:55:47 +0200 Subject: [PATCH 2/3] test: stabilize trace usage assertions --- plugins/tracing/test/trace.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index 6f0731f..ca9890a 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -85,7 +85,9 @@ describe("convertRollout", () => { // Two generations, both children of the root, named "LLM" (the model name // lives in the model attribute, not the observation name). - const generations = spans.filter((s) => obsType(s) === "generation"); + const generations = spans + .filter((s) => obsType(s) === "generation") + .sort((a, b) => startMs(a) - startMs(b)); expect(generations).toHaveLength(2); for (const gen of generations) { expect(gen.name).toBe("LLM"); @@ -94,9 +96,11 @@ describe("convertRollout", () => { } // Usage is sent in Langfuse's OpenAI-compatible shape. Langfuse then // normalizes the inclusive parent counts and nested detail counts. - const usages = generations.map((g) => - JSON.parse(attr(g, "langfuse.observation.usage_details")), - ); + const usages = generations.map((generation) => { + const usage = attr(generation, "langfuse.observation.usage_details"); + expect(usage, "expected generation usage details").not.toBe(""); + return JSON.parse(usage); + }); expect(usages).toEqual([ { prompt_tokens: 100, From 3242f2785324c5ebe97fece1301bb2696cb1e4e1 Mon Sep 17 00:00:00 2001 From: Valery Meleshkin Date: Thu, 6 Aug 2026 15:30:59 +0200 Subject: [PATCH 3/3] fix: log dropped token usage --- plugins/tracing/dist/index.mjs | 10 ++++++++-- plugins/tracing/src/trace.ts | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/tracing/dist/index.mjs b/plugins/tracing/dist/index.mjs index d74aa5a..f96eb74 100644 --- a/plugins/tracing/dist/index.mjs +++ b/plugins/tracing/dist/index.mjs @@ -46988,8 +46988,14 @@ function isTokenCount(value) { function toUsageDetails(usage) { if (!usage) return void 0; const { input_tokens: input, output_tokens: output, total_tokens: total, cached_input_tokens: cached$1, reasoning_output_tokens: reasoning } = usage; - if (!isTokenCount(input) || !isTokenCount(output) || !isTokenCount(total) || total !== input + output) return; - if (cached$1 !== void 0 && (!isTokenCount(cached$1) || cached$1 > input) || reasoning !== void 0 && (!isTokenCount(reasoning) || reasoning > output)) return; + if (!isTokenCount(input) || !isTokenCount(output) || !isTokenCount(total) || total !== input + output) { + debugLog("dropping usage: missing or inconsistent token counts", usage); + return; + } + if (cached$1 !== void 0 && (!isTokenCount(cached$1) || cached$1 > input) || reasoning !== void 0 && (!isTokenCount(reasoning) || reasoning > output)) { + debugLog("dropping usage: implausible cached/reasoning details", usage); + return; + } return { prompt_tokens: input, completion_tokens: output, diff --git a/plugins/tracing/src/trace.ts b/plugins/tracing/src/trace.ts index ffb68c7..24b4f0e 100644 --- a/plugins/tracing/src/trace.ts +++ b/plugins/tracing/src/trace.ts @@ -133,12 +133,14 @@ function toUsageDetails( !isTokenCount(total) || total !== input + output ) { + debugLog("dropping usage: missing or inconsistent token counts", usage); return undefined; } if ( (cached !== undefined && (!isTokenCount(cached) || cached > input)) || (reasoning !== undefined && (!isTokenCount(reasoning) || reasoning > output)) ) { + debugLog("dropping usage: implausible cached/reasoning details", usage); return undefined; }