diff --git a/TELEMETRY.md b/TELEMETRY.md index f27e546e1a..92b56e26cb 100644 --- a/TELEMETRY.md +++ b/TELEMETRY.md @@ -189,6 +189,7 @@ Attributes: `gen_ai.operation.name`, `gen_ai.request.model`, `gen_ai.usage.output_tokens`, `gen_ai.usage.input_tokens.cached`, `gen_ai.usage.input_tokens.cache_write`, `app.ai.reasoning_tokens`, `app.ai.empty_output.attempt`, `app.ai.provider_error.kind`, +`app.ai.provider_error.summary`, `app.ai.provider_error.retry_attempt`, `app.guardian.review_attempt`, `app.ai.cost.input_usd`, `app.ai.cost.output_usd`, `app.ai.cost.cache_read_usd`, `app.ai.cost.cache_write_usd`, diff --git a/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts b/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts index 740ffc1eff..af44285339 100644 --- a/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts +++ b/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts @@ -89,7 +89,7 @@ describe("eval AI Gateway dispatcher", () => { failure: failedAssistant, messages: [userMessage as PiMessage, failedAssistant as PiMessage], }), - ).toEqual({ delayMs: 2_000, messages: [userMessage as PiMessage] }); + ).toMatchObject({ delayMs: 2_000, messages: [userMessage as PiMessage] }); } finally { await restore(); } diff --git a/packages/junior/src/chat/agent/index.ts b/packages/junior/src/chat/agent/index.ts index 4015ad7537..4f3fe0c68d 100644 --- a/packages/junior/src/chat/agent/index.ts +++ b/packages/junior/src/chat/agent/index.ts @@ -1604,7 +1604,10 @@ async function executeAgentRunInPrivacyContext( providerRetryAttempt += 1; await prepareRetry(providerRetry.messages); - logWarn("agent.turn.provider.retrying"); + logWarn("agent.turn.provider.retrying", { + ...getProviderErrorAttributes(providerRetry.providerError), + "app.ai.provider_error.retry_attempt": providerRetryAttempt, + }); await sleep(providerRetry.delayMs, signal); run = agent!.continue(); } diff --git a/packages/junior/src/chat/pi/traced-stream.ts b/packages/junior/src/chat/pi/traced-stream.ts index c58bb6abd3..a7907b7ad6 100644 --- a/packages/junior/src/chat/pi/traced-stream.ts +++ b/packages/junior/src/chat/pi/traced-stream.ts @@ -25,6 +25,10 @@ import { toGenAiMessagesTraceAttributes, } from "@/chat/conversation-privacy"; import { hasCompactedConversationContext } from "@/chat/services/context-compaction-marker"; +import { + createProviderError, + getProviderErrorAttributes, +} from "@/chat/services/provider-error"; type GenAiAttributeMode = "content" | "metadata"; type TraceAttributeValue = string | number | boolean | string[]; @@ -126,6 +130,12 @@ function buildChatEndAttributes( attributes["gen_ai.response.model"] = message.model; } + if (message.stopReason === "error") { + const providerError = createProviderError(message.errorMessage ?? ""); + Object.assign(attributes, getProviderErrorAttributes(providerError)); + attributes["error.type"] = providerError.kind; + } + return attributes; } @@ -184,14 +194,17 @@ export function createTracedStreamFn( .then( (finalMessage: AssistantMessage) => { try { - for (const [key, value] of Object.entries( - buildChatEndAttributes(finalMessage, mode), - )) { + const endAttributes = buildChatEndAttributes(finalMessage, mode); + for (const [key, value] of Object.entries(endAttributes)) { span.setAttribute(key, value); } if (finalMessage.stopReason === "error") { - span.setAttribute("error.type", "provider_error"); - span.setStatus({ code: 2, message: "LLM stream failed" }); + const summary = endAttributes["app.ai.provider_error.summary"]; + span.setStatus({ + code: 2, + message: + typeof summary === "string" ? summary : "LLM stream failed", + }); } } finally { span.end(); diff --git a/packages/junior/src/chat/services/provider-error.ts b/packages/junior/src/chat/services/provider-error.ts index 146b7a913b..337b9f7a71 100644 --- a/packages/junior/src/chat/services/provider-error.ts +++ b/packages/junior/src/chat/services/provider-error.ts @@ -116,6 +116,24 @@ function providerMessage(error: unknown): string { return (error instanceof Error ? error.message : String(error)).trim(); } +const PROVIDER_ERROR_SUMMARY_MAX_CHARS = 240; + +/** Short provider-boundary text for telemetry. Drop trailing JSON bodies. */ +function summarizeProviderErrorMessage(message: string): string | undefined { + const normalized = message.trim().replace(/\s+/g, " "); + if (!normalized) return undefined; + + // Gateway errors often look like `503 {"error":...}`. Keep the lead text. + const jsonStart = normalized.indexOf("{"); + const summary = ( + jsonStart >= 0 ? normalized.slice(0, jsonStart) : normalized + ).trim(); + if (!summary) return undefined; + return summary.length > PROVIDER_ERROR_SUMMARY_MAX_CHARS + ? `${summary.slice(0, PROVIDER_ERROR_SUMMARY_MAX_CHARS)}...` + : summary; +} + function extractTransportKind( error: unknown, depth = 0, @@ -284,6 +302,16 @@ export function getProviderErrorUserMessage(error: ProviderError): string { export function getProviderErrorAttributes( error: ProviderError, ): Record { + const cause = error.cause; + const causeMessage = + cause instanceof Error + ? cause.message + : typeof cause === "string" + ? cause + : undefined; + const summary = causeMessage + ? summarizeProviderErrorMessage(causeMessage) + : undefined; return { "app.ai.provider_error.kind": error.kind, "app.ai.provider_error.retryable": error.retryable, @@ -293,6 +321,7 @@ export function getProviderErrorAttributes( ...(error.retryAfterMs !== undefined ? { "app.ai.provider_error.retry_after_ms": error.retryAfterMs } : undefined), + ...(summary ? { "app.ai.provider_error.summary": summary } : undefined), ...(error.modelId ? { "gen_ai.request.model": error.modelId } : undefined), }; } diff --git a/packages/junior/src/chat/services/provider-retry.ts b/packages/junior/src/chat/services/provider-retry.ts index 1c0098fd7f..e68b674c32 100644 --- a/packages/junior/src/chat/services/provider-retry.ts +++ b/packages/junior/src/chat/services/provider-retry.ts @@ -3,7 +3,10 @@ import { type AssistantMessage, } from "@earendil-works/pi-ai"; import type { PiMessage } from "@/chat/pi/messages"; -import { createProviderError } from "@/chat/services/provider-error"; +import { + createProviderError, + type ProviderError, +} from "@/chat/services/provider-error"; import { getPiMessageRole, trimTrailingAssistantMessages, @@ -17,7 +20,13 @@ export function nextProviderRetry(args: { attempt: number; failure?: AssistantMessage; messages: PiMessage[]; -}): { delayMs: number; messages: PiMessage[] } | undefined { +}): + | { + delayMs: number; + messages: PiMessage[]; + providerError: ProviderError; + } + | undefined { const backoffMs = PROVIDER_RETRY_DELAYS_MS[args.attempt]; const errorMessage = args.failure?.errorMessage; if (backoffMs === undefined || !args.failure || !errorMessage) { @@ -47,5 +56,5 @@ export function nextProviderRetry(args: { return undefined; } - return { delayMs, messages }; + return { delayMs, messages, providerError }; } diff --git a/packages/junior/tests/unit/services/provider-retry.test.ts b/packages/junior/tests/unit/services/provider-retry.test.ts index 7a8164da10..559f9645ed 100644 --- a/packages/junior/tests/unit/services/provider-retry.test.ts +++ b/packages/junior/tests/unit/services/provider-retry.test.ts @@ -5,6 +5,7 @@ import type { PiMessage } from "@/chat/pi/messages"; import { createProviderError, findProviderError, + getProviderErrorAttributes, getProviderErrorUserMessage, isProviderRetryError, ProviderError, @@ -34,6 +35,11 @@ describe("provider retry helpers", () => { kind: "network", retryable: true, }); + expect(getProviderErrorAttributes(error)).toMatchObject({ + "app.ai.provider_error.kind": "network", + "app.ai.provider_error.summary": + "Anthropic stream ended before message_stop", + }); expect(isProviderRetryError(error)).toBe(true); expect(isProviderRetryError(createProviderError("invalid_api_key"))).toBe( false, @@ -42,6 +48,21 @@ describe("provider retry helpers", () => { expect(isProviderRetryError(new Error(error.message))).toBe(false); }); + it("keeps a bounded provider summary without gateway JSON payloads", () => { + expect( + getProviderErrorAttributes( + createProviderError(XAI_SERVICE_UNAVAILABLE, { + modelId: "xai/grok-4.5", + }), + ), + ).toMatchObject({ + "app.ai.provider_error.kind": "server", + "app.ai.provider_error.status": 503, + "app.ai.provider_error.summary": "503", + "gen_ai.request.model": "xai/grok-4.5", + }); + }); + it("finds provider errors preserved by domain wrappers", () => { const providerError = createProviderError("No object generated", { kind: "invalid_response", @@ -114,13 +135,21 @@ describe("provider retry helpers", () => { "Anthropic stream ended before message_stop", ); - expect( - nextProviderRetry({ - attempt: 0, - failure: failedAssistant, - messages: [user, failedAssistant], - }), - ).toEqual({ delayMs: 2_000, messages: [user] }); + const retry = nextProviderRetry({ + attempt: 0, + failure: failedAssistant, + messages: [user, failedAssistant], + }); + expect(retry).toMatchObject({ + delayMs: 2_000, + messages: [user], + providerError: { kind: "network", retryable: true }, + }); + expect(getProviderErrorAttributes(retry!.providerError)).toMatchObject({ + "app.ai.provider_error.kind": "network", + "app.ai.provider_error.summary": + "Anthropic stream ended before message_stop", + }); }); it("retries a structured xAI 503 despite gateway credential metadata", () => { @@ -147,7 +176,7 @@ describe("provider retry helpers", () => { failure: failedAssistant, messages: [user, failedAssistant], }), - ).toEqual({ delayMs: 2_000, messages: [user] }); + ).toMatchObject({ delayMs: 2_000, messages: [user] }); }); it("honors bounded rate-limit hints", () => { @@ -175,7 +204,7 @@ describe("provider retry helpers", () => { failure: failedAssistant, messages: [user, failedAssistant], }), - ).toEqual({ delayMs: 60_000, messages: [user] }); + ).toMatchObject({ delayMs: 60_000, messages: [user] }); }); it("classifies HTTP request timeouts without overriding permanent signals", () => { @@ -232,7 +261,7 @@ describe("provider retry helpers", () => { failure, messages: [user, failure], }), - ).toEqual({ delayMs: 2_000, messages: [user] }); + ).toMatchObject({ delayMs: 2_000, messages: [user] }); }); it("keeps explicit permanent request failures terminal", () => { diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index 4346fab2a6..86132ca9aa 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -72,10 +72,14 @@ describe("finalizeFailedTurnReply", () => { "app.ai.provider_error.kind": "server", "app.ai.provider_error.retryable": true, "app.ai.provider_error.status": 503, + "app.ai.provider_error.summary": "503", "app.ai.failure_reason": "server", "gen_ai.request.model": "xai/grok-4.5", }); expect(attributes).not.toHaveProperty("exception.message"); + expect(String(attributes?.["app.ai.provider_error.summary"] ?? "")).not.toContain( + "providerMetadata", + ); }); it("classifies empty execution failures without raw exception text", () => {