diff --git a/packages/junior-dashboard/src/client/conversations/ConversationTranscript.tsx b/packages/junior-dashboard/src/client/conversations/ConversationTranscript.tsx index 2d6184587..44e033afc 100644 --- a/packages/junior-dashboard/src/client/conversations/ConversationTranscript.tsx +++ b/packages/junior-dashboard/src/client/conversations/ConversationTranscript.tsx @@ -167,7 +167,8 @@ const VisibleTranscriptEntries = memo(function VisibleTranscriptEntries(props: { )} renderFailure={(entry) => ( )} @@ -374,7 +375,8 @@ function RedactedTranscriptView(props: { )} renderFailure={(entry) => ( )} diff --git a/packages/junior-dashboard/src/client/conversations/TranscriptFailureView.tsx b/packages/junior-dashboard/src/client/conversations/TranscriptFailureView.tsx index 691ff2040..df161726a 100644 --- a/packages/junior-dashboard/src/client/conversations/TranscriptFailureView.tsx +++ b/packages/junior-dashboard/src/client/conversations/TranscriptFailureView.tsx @@ -1,20 +1,28 @@ import { CircleAlert } from "lucide-react"; +import type { + ConversationTurnFailureCode, + ConversationTurnFailureReason, +} from "@sentry/junior/api/schema"; -import { getDashboardAgentName } from "../agentName"; import { formatMessageTimestamp } from "../format"; +import { + transcriptFailureDescription, + transcriptFailureTitle, +} from "./transcriptFailure"; /** Render a terminal transcript failure as a distinct alert surface. */ export function TranscriptFailureView(props: { - outcome: "error" | "delivery_failed"; + failureCode: ConversationTurnFailureCode; + failureReason?: ConversationTurnFailureReason; timestamp?: number; }) { const timestamp = formatMessageTimestamp(props.timestamp); - const deliveryFailed = props.outcome === "delivery_failed"; return (
- {deliveryFailed ? "Message delivery failed" : "Agent response failed"} + {transcriptFailureTitle(props.failureCode, props.failureReason)}
- {deliveryFailed - ? `${getDashboardAgentName()} could not deliver this message to its destination.` - : `The model response ended before ${getDashboardAgentName()} could complete this turn.`} + {transcriptFailureDescription(props.failureCode, props.failureReason)}
{timestamp ? ( diff --git a/packages/junior-dashboard/src/client/conversations/eventTranscript.ts b/packages/junior-dashboard/src/client/conversations/eventTranscript.ts index 058d7203f..8ec8b851c 100644 --- a/packages/junior-dashboard/src/client/conversations/eventTranscript.ts +++ b/packages/junior-dashboard/src/client/conversations/eventTranscript.ts @@ -413,8 +413,11 @@ export function transcriptMessagesFromEvents( if (data.type === "turn_lifecycle" && data.state === "failed") { messages.push({ - role: data.failureKind === "delivery" ? "system" : "assistant", - outcome: data.failureKind === "delivery" ? "delivery_failed" : "error", + role: data.failureCode === "delivery_failed" ? "system" : "assistant", + failureCode: data.failureCode, + ...(data.failureReason + ? { failureReason: data.failureReason } + : undefined), parts: [], sourceSeq: event.seq, timestamp: eventTimestamp(event), diff --git a/packages/junior-dashboard/src/client/conversations/transcriptBottomPinning.ts b/packages/junior-dashboard/src/client/conversations/transcriptBottomPinning.ts index 6b9539da2..2534174cb 100644 --- a/packages/junior-dashboard/src/client/conversations/transcriptBottomPinning.ts +++ b/packages/junior-dashboard/src/client/conversations/transcriptBottomPinning.ts @@ -171,7 +171,8 @@ function eventTailVersion(event: ConversationReportEvent | undefined): string { data.type, data.turnId, data.state, - "failureKind" in data ? data.failureKind : "", + "failureCode" in data ? data.failureCode : "", + "failureReason" in data ? (data.failureReason ?? "") : "", ].join(":"); case "structured_event": return [ diff --git a/packages/junior-dashboard/src/client/conversations/transcriptFailure.ts b/packages/junior-dashboard/src/client/conversations/transcriptFailure.ts new file mode 100644 index 000000000..a95c2f8bf --- /dev/null +++ b/packages/junior-dashboard/src/client/conversations/transcriptFailure.ts @@ -0,0 +1,129 @@ +import type { + ConversationTurnFailureCode, + ConversationTurnFailureReason, +} from "@sentry/junior/api/schema"; + +import { getDashboardAgentName } from "../agentName"; + +/** Title for one failed turn in the transcript. */ +export function transcriptFailureTitle( + failureCode: ConversationTurnFailureCode, + failureReason?: ConversationTurnFailureReason, +): string { + if (failureCode === "model_execution_failed" && failureReason) { + switch (failureReason) { + case "auth": + return "Model credentials rejected"; + case "permission": + return "Model access denied"; + case "rate_limit": + return "Model rate limited"; + case "capacity": + return "Model at capacity"; + case "timeout": + return "Model timed out"; + case "network": + return "Model connection failed"; + case "server": + return "Model service error"; + case "invalid_request": + return "Invalid model request"; + case "invalid_response": + return "Invalid model response"; + case "quota": + return "Model quota exhausted"; + case "content_policy": + return "Blocked by content policy"; + case "empty_output": + return "Empty model response"; + case "tool_errors": + return "Tool failed"; + case "suppressed_output": + return "Model output was dropped"; + case "unknown": + return "Model service error"; + } + } + + switch (failureCode) { + case "delivery_failed": + return "Message delivery failed"; + case "model_execution_failed": + return "Model failed"; + case "persistence_failed": + return "Save failed"; + case "agent_run_failed": + return "Internal error"; + } + + return "Internal error"; +} + +/** Short description for one failed turn in the transcript. */ +export function transcriptFailureDescription( + failureCode: ConversationTurnFailureCode, + failureReason?: ConversationTurnFailureReason, +): string { + const agentName = getDashboardAgentName(); + + if (failureCode === "model_execution_failed" && failureReason) { + switch (failureReason) { + case "auth": + return "The model service rejected credentials. An admin needs to fix configuration."; + case "permission": + return "The model service denied access to this model or request."; + case "rate_limit": + return "The model is rate limited. Try again shortly."; + case "capacity": + return "The selected model is at capacity. Try again shortly."; + case "timeout": + return "The model service timed out before the turn finished."; + case "network": + return "The model service had a connection problem before the turn finished."; + case "server": + return "The model service returned an error."; + case "invalid_request": + return "The model service rejected this request as invalid."; + case "invalid_response": + return "The model service returned an invalid response."; + case "quota": + return "The model service quota is exhausted. An admin needs to fix billing or limits."; + case "content_policy": + return "The model service blocked this request under its content policy."; + case "empty_output": + return `The model returned no usable text before ${agentName} finished this turn.`; + case "tool_errors": + return "A tool failed and the turn could not finish."; + case "suppressed_output": + return `The model produced text that ${agentName} could not deliver.`; + case "unknown": + return "The model service failed for an unknown reason."; + } + } + + switch (failureCode) { + case "delivery_failed": + return `${agentName} could not deliver this message.`; + case "model_execution_failed": + return `The model stopped before ${agentName} finished this turn.`; + case "persistence_failed": + return `${agentName} could not save the result of this turn.`; + case "agent_run_failed": + return `${agentName} hit an internal error during this turn.`; + } + + return `${agentName} hit an internal error during this turn.`; +} + +/** Search text for one failed turn in the transcript. */ +export function transcriptFailureSearchText( + failureCode: ConversationTurnFailureCode, + failureReason?: ConversationTurnFailureReason, +): string { + return [ + transcriptFailureTitle(failureCode, failureReason), + transcriptFailureDescription(failureCode, failureReason), + failureCode.replaceAll("_", " "), + ...(failureReason ? [failureReason.replaceAll("_", " ")] : []), + ].join(" "); +} diff --git a/packages/junior-dashboard/src/client/conversations/transcriptRenderModel.ts b/packages/junior-dashboard/src/client/conversations/transcriptRenderModel.ts index 223467e6d..d0f43fdf8 100644 --- a/packages/junior-dashboard/src/client/conversations/transcriptRenderModel.ts +++ b/packages/junior-dashboard/src/client/conversations/transcriptRenderModel.ts @@ -1,3 +1,8 @@ +import type { + ConversationTurnFailureCode, + ConversationTurnFailureReason, +} from "@sentry/junior/api/schema"; + import type { TranscriptViewAttachmentsDeliveredPart, TranscriptViewContextEventPart, @@ -12,7 +17,8 @@ import type { export type RenderedFailureEntry = { key: string; kind: "failure"; - outcome: "error" | "delivery_failed"; + failureCode: ConversationTurnFailureCode; + failureReason?: ConversationTurnFailureReason; timestamp?: number; }; @@ -157,11 +163,14 @@ export function groupTranscriptMessages( } flushMessage(); - if (message.outcome) { + if (message.failureCode) { entries.push({ key: `${message.sourceSeq}:failure`, kind: "failure", - outcome: message.outcome, + failureCode: message.failureCode, + ...(message.failureReason + ? { failureReason: message.failureReason } + : undefined), timestamp: message.timestamp, }); } diff --git a/packages/junior-dashboard/src/client/conversations/transcriptSearch.tsx b/packages/junior-dashboard/src/client/conversations/transcriptSearch.tsx index a73236f88..32e135ef3 100644 --- a/packages/junior-dashboard/src/client/conversations/transcriptSearch.tsx +++ b/packages/junior-dashboard/src/client/conversations/transcriptSearch.tsx @@ -5,6 +5,7 @@ import { messageRawText, type RenderedTranscriptEntry, } from "./transcriptRenderModel"; +import { transcriptFailureSearchText } from "./transcriptFailure"; import { stringifyPartValue } from "../format"; // ─── Context ──────────────────────────────────────────────────────────────── @@ -131,9 +132,7 @@ export function entryMatchesSearch( if (entry.kind === "failure") { return textContains( - entry.outcome === "delivery_failed" - ? "message delivery failed" - : "agent response failed error", + transcriptFailureSearchText(entry.failureCode, entry.failureReason), normalizedQuery, ); } diff --git a/packages/junior-dashboard/src/client/markdownExport.ts b/packages/junior-dashboard/src/client/markdownExport.ts index 2224f1e44..e3f8e2333 100644 --- a/packages/junior-dashboard/src/client/markdownExport.ts +++ b/packages/junior-dashboard/src/client/markdownExport.ts @@ -14,8 +14,15 @@ import { groupTranscriptMessages, messageRawText, } from "./conversations/transcriptRenderModel"; -import { getDashboardAgentName } from "./agentName"; import { conversationTranscriptMessages } from "./conversations/eventTranscript"; +import { + transcriptFailureDescription, + transcriptFailureTitle, +} from "./conversations/transcriptFailure"; +import type { + ConversationTurnFailureCode, + ConversationTurnFailureReason, +} from "@sentry/junior/api/schema"; import type { Conversation, ConversationTranscript, @@ -126,7 +133,8 @@ function appendTranscriptMessages( appendFailure( lines, conversationTranscript, - entry.outcome, + entry.failureCode, + entry.failureReason, entry.timestamp, ); continue; @@ -213,22 +221,20 @@ function appendReasoning( function appendFailure( lines: string[], conversationTranscript: ConversationTranscript, - outcome: "error" | "delivery_failed", + failureCode: ConversationTurnFailureCode, + failureReason: ConversationTurnFailureReason | undefined, timestamp: number | undefined, ): void { lines.push( "", - outcome === "delivery_failed" - ? "### Message delivery failed" - : "### Agent response failed", + `### ${transcriptFailureTitle(failureCode, failureReason)}`, ); addEventMeta(lines, conversationTranscript, timestamp); - lines.push( - "", - outcome === "delivery_failed" - ? `${getDashboardAgentName()} could not deliver this message to its destination.` - : `The model response ended before ${getDashboardAgentName()} could complete this turn.`, - ); + lines.push("", transcriptFailureDescription(failureCode, failureReason)); + addMetaLine(lines, "Code", failureCode); + if (failureReason) { + addMetaLine(lines, "Reason", failureReason); + } } function appendContextEvent( diff --git a/packages/junior-dashboard/src/client/types.ts b/packages/junior-dashboard/src/client/types.ts index 5f1c31a20..23937e0f4 100644 --- a/packages/junior-dashboard/src/client/types.ts +++ b/packages/junior-dashboard/src/client/types.ts @@ -8,6 +8,8 @@ import type { ActorIdentity, ConversationStatsReport, ConversationSummaryReport, + ConversationTurnFailureCode, + ConversationTurnFailureReason, } from "@sentry/junior/api/schema"; import type { ConversationDetailReport } from "@sentry/junior/api/schema"; import type { ConversationEventPresentation } from "@sentry/junior-plugin-api"; @@ -128,7 +130,8 @@ export type TranscriptViewMessage = { reasoningLevel: string; source: "configured" | "inherited" | "router"; }; - outcome?: "error" | "delivery_failed"; + failureCode?: ConversationTurnFailureCode; + failureReason?: ConversationTurnFailureReason; parts: TranscriptViewPart[]; role: "assistant" | "system" | "tool" | "user"; source?: "slack" | "web"; diff --git a/packages/junior-dashboard/src/mock-reporting/fixtures.ts b/packages/junior-dashboard/src/mock-reporting/fixtures.ts index cc9840170..1c47e3f12 100644 --- a/packages/junior-dashboard/src/mock-reporting/fixtures.ts +++ b/packages/junior-dashboard/src/mock-reporting/fixtures.ts @@ -1185,7 +1185,8 @@ function failedConversation(nowMs: number): ConversationDetailReport { type: "turn_lifecycle", turnId: "failed-turn", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", + failureReason: "network", }), ], }); diff --git a/packages/junior-dashboard/tests/markdownExport.test.ts b/packages/junior-dashboard/tests/markdownExport.test.ts index c933d9ea9..d1d62d7ff 100644 --- a/packages/junior-dashboard/tests/markdownExport.test.ts +++ b/packages/junior-dashboard/tests/markdownExport.test.ts @@ -268,7 +268,8 @@ describe("dashboard canonical-event Markdown export", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", + failureReason: "network", }), ]), ); @@ -292,7 +293,9 @@ describe("dashboard canonical-event Markdown export", () => { expect(markdown).toContain("- Model: openai/gpt-5-mini"); expect(markdown).toContain("- Reasoning: medium"); expect(markdown).toContain("Investigate the remaining deployment failure."); - expect(markdown).toContain("### Agent response failed"); + expect(markdown).toContain("### Model connection failed"); + expect(markdown).toContain("- Code: model_execution_failed"); + expect(markdown).toContain("- Reason: network"); expect(markdown).not.toContain("missing"); expect(markdown).not.toContain("Result: running"); }); @@ -304,17 +307,16 @@ describe("dashboard canonical-event Markdown export", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "delivery", + failureCode: "delivery_failed", }), ]), ); expect(markdown).toContain("### Message delivery failed"); - expect(markdown).toContain( - "Junior could not deliver this message to its destination.", - ); + expect(markdown).toContain("Junior could not deliver this message."); expect(markdown).not.toContain("turn-1"); - expect(markdown).not.toContain("Agent response failed"); + expect(markdown).not.toContain("Model connection failed"); + expect(markdown).not.toContain("Internal error"); }); it("labels redacted in-progress tools without inventing a completion", () => { diff --git a/packages/junior-dashboard/tests/telemetry-components.test.tsx b/packages/junior-dashboard/tests/telemetry-components.test.tsx index ca08fc7be..59fe601f2 100644 --- a/packages/junior-dashboard/tests/telemetry-components.test.tsx +++ b/packages/junior-dashboard/tests/telemetry-components.test.tsx @@ -688,13 +688,15 @@ describe("dashboard canonical-event components", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", + failureReason: "network", }), ]), ); expect(html).toContain("Context compacted"); expect(html).toContain("Model handoff"); - expect(html).toContain("Agent response failed"); + expect(html).toContain("Model connection failed"); + expect(html).toContain('data-transcript-failure-reason="network"'); }); it("anchors structured events to the transcript rail", () => { @@ -809,15 +811,14 @@ describe("dashboard canonical-event components", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "delivery", + failureCode: "delivery_failed", }), ]), ); expect(html).toContain("Message delivery failed"); - expect(html).toContain( - "Junior could not deliver this message to its destination.", - ); - expect(html).not.toContain("Agent response failed"); + expect(html).toContain("Junior could not deliver this message."); + expect(html).not.toContain("Model connection failed"); + expect(html).not.toContain("Internal error"); }); it("does not invent an object for an empty raw message", () => { diff --git a/packages/junior-dashboard/tests/transcriptActivityGroup.test.ts b/packages/junior-dashboard/tests/transcriptActivityGroup.test.ts index a69575ead..08a9fc322 100644 --- a/packages/junior-dashboard/tests/transcriptActivityGroup.test.ts +++ b/packages/junior-dashboard/tests/transcriptActivityGroup.test.ts @@ -108,7 +108,7 @@ function failure(): RenderedTranscriptEntry { return { key: "failure:1", kind: "failure", - outcome: "error", + failureCode: "model_execution_failed", timestamp: 1, }; } diff --git a/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts b/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts index 8212072b7..9bccad023 100644 --- a/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts +++ b/packages/junior-dashboard/tests/transcriptBottomPinning.test.ts @@ -327,7 +327,7 @@ describe("transcript bottom pinning", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", }, }, ], diff --git a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts index 4865f510f..eb12408d1 100644 --- a/packages/junior-dashboard/tests/transcriptRenderModel.test.ts +++ b/packages/junior-dashboard/tests/transcriptRenderModel.test.ts @@ -626,7 +626,7 @@ describe("canonical event transcript reduction", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", }), ]), ); @@ -860,13 +860,13 @@ describe("canonical event transcript reduction", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", }), event(7, "2026-01-01T00:00:07.000Z", { type: "turn_lifecycle", turnId: "turn-2", state: "failed", - failureKind: "delivery", + failureCode: "delivery_failed", }), ]), ), @@ -902,7 +902,7 @@ describe("transcript render grouping", () => { const messages: TranscriptViewMessage[] = [ { role: "assistant", - outcome: "error", + failureCode: "model_execution_failed", sourceSeq: 42, timestamp: 1_000, parts: [], @@ -912,7 +912,7 @@ describe("transcript render grouping", () => { { key: "42:failure", kind: "failure", - outcome: "error", + failureCode: "model_execution_failed", timestamp: 1_000, }, ]); diff --git a/packages/junior/src/api/conversations/events.ts b/packages/junior/src/api/conversations/events.ts index a9a3d30a7..95b7230e3 100644 --- a/packages/junior/src/api/conversations/events.ts +++ b/packages/junior/src/api/conversations/events.ts @@ -401,8 +401,10 @@ function reportEventData(args: { type: "turn_lifecycle", turnId: data.turnId, state: "failed", - failureKind: - data.failureCode === "delivery_failed" ? "delivery" : "agent", + failureCode: data.failureCode, + ...(data.failureReason + ? { failureReason: data.failureReason } + : undefined), }; case "compaction": return { diff --git a/packages/junior/src/api/schema.ts b/packages/junior/src/api/schema.ts index 9a25aa20b..74d5ff78e 100644 --- a/packages/junior/src/api/schema.ts +++ b/packages/junior/src/api/schema.ts @@ -36,6 +36,8 @@ export { conversationPendingMessagesReportSchema, conversationReportEventDataSchema, conversationReportEventSchema, + conversationTurnFailureCodeSchema, + conversationTurnFailureReasonSchema, conversationStatsReportSchema, conversationSummaryReportSchema, createConversationBodySchema, @@ -60,6 +62,8 @@ export type { ConversationReportEvent, ConversationReportEventData, ConversationReportStatus, + ConversationTurnFailureCode, + ConversationTurnFailureReason, ConversationMetricDay, ConversationModelUsage, ConversationParams, diff --git a/packages/junior/src/api/schema/conversation.ts b/packages/junior/src/api/schema/conversation.ts index 4bfff922e..da2a1d587 100644 --- a/packages/junior/src/api/schema/conversation.ts +++ b/packages/junior/src/api/schema/conversation.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + conversationTurnFailureCodeSchema, + conversationTurnFailureReasonSchema, +} from "@/chat/conversations/history"; import { usageCostSchema, usageSchema } from "@/usage-schema"; import { conversationAnnotationInputSchema, @@ -6,6 +10,11 @@ import { conversationEventPresentationSchema, } from "@sentry/junior-plugin-api"; +export { + conversationTurnFailureCodeSchema, + conversationTurnFailureReasonSchema, +}; + export const conversationReportStatusSchema = z.enum([ "active", "completed", @@ -423,7 +432,8 @@ const conversationReportTurnLifecycleEventDataSchema = z.discriminatedUnion( type: z.literal("turn_lifecycle"), turnId: z.string().min(1), state: z.literal("failed"), - failureKind: z.enum(["agent", "delivery"]), + failureCode: conversationTurnFailureCodeSchema, + failureReason: conversationTurnFailureReasonSchema.optional(), }) .strict(), ], @@ -791,6 +801,12 @@ export const conversationStatsReportSchema = z export type ConversationReportStatus = z.infer< typeof conversationReportStatusSchema >; +export type ConversationTurnFailureCode = z.infer< + typeof conversationTurnFailureCodeSchema +>; +export type ConversationTurnFailureReason = z.infer< + typeof conversationTurnFailureReasonSchema +>; export type ConversationSurface = z.infer; export type ConversationCost = z.infer; export type ConversationUsage = z.infer; diff --git a/packages/junior/src/chat/api-turns/work.ts b/packages/junior/src/chat/api-turns/work.ts index 6a974193c..3f45993f0 100644 --- a/packages/junior/src/chat/api-turns/work.ts +++ b/packages/junior/src/chat/api-turns/work.ts @@ -787,6 +787,7 @@ export function createApiTurnWorker( reply: result, logException, }); + const modelFailureReason = finalized.failureReason; const reply = finalized.reply; modelFailureEventId = finalized.eventId; if (reply.diagnostics.outcome !== "success") { @@ -836,6 +837,9 @@ export function createApiTurnWorker( ? { eventId: modelFailureEventId } : undefined), failureCode: "model_execution_failed", + ...(modelFailureReason + ? { failureReason: modelFailureReason } + : undefined), outcome: "failed", }; }, diff --git a/packages/junior/src/chat/conversations/history.ts b/packages/junior/src/chat/conversations/history.ts index dbb53f376..45ed2ed47 100644 --- a/packages/junior/src/chat/conversations/history.ts +++ b/packages/junior/src/chat/conversations/history.ts @@ -236,7 +236,7 @@ export const conversationTurnSurfaceSchema = z.enum([ "internal", ]); -/** Stable, privacy-safe classification for a failed turn. */ +/** Where a failed turn stopped. */ export const conversationTurnFailureCodeSchema = z.enum([ "agent_run_failed", "delivery_failed", @@ -244,11 +244,38 @@ export const conversationTurnFailureCodeSchema = z.enum([ "persistence_failed", ]); -/** Failure classification persisted without raw provider or exception data. */ +/** Where a failed turn stopped. */ export type ConversationTurnFailureCode = z.output< typeof conversationTurnFailureCodeSchema >; +/** + * Why a failed turn stopped. + * Values are fixed labels only. Do not store raw exception text. + */ +export const conversationTurnFailureReasonSchema = z.enum([ + "auth", + "permission", + "rate_limit", + "capacity", + "timeout", + "network", + "server", + "invalid_request", + "invalid_response", + "quota", + "content_policy", + "unknown", + "empty_output", + "tool_errors", + "suppressed_output", +]); + +/** Why a failed turn stopped. */ +export type ConversationTurnFailureReason = z.output< + typeof conversationTurnFailureReasonSchema +>; + const turnStartedEventDataSchema = z .object({ type: z.literal("turn_started"), @@ -331,6 +358,7 @@ const turnFailedEventDataSchema = z type: z.literal("turn_failed"), turnId: z.string().min(1), failureCode: conversationTurnFailureCodeSchema, + failureReason: conversationTurnFailureReasonSchema.optional(), eventId: z .string() .regex(/^[a-f0-9]{32}$/i) diff --git a/packages/junior/src/chat/conversations/turn-lifecycle.ts b/packages/junior/src/chat/conversations/turn-lifecycle.ts index 66b2ff6a6..31f3761df 100644 --- a/packages/junior/src/chat/conversations/turn-lifecycle.ts +++ b/packages/junior/src/chat/conversations/turn-lifecycle.ts @@ -1,6 +1,7 @@ import type { ConversationEventStore, ConversationTurnFailureCode, + ConversationTurnFailureReason, } from "./history"; /** Product-owned inputs for opening one correlated conversation turn. */ @@ -20,12 +21,13 @@ export interface CompleteConversationTurnInput { turnId: string; } -/** Privacy-safe inputs for failing one correlated conversation turn. */ +/** Inputs for failing one correlated conversation turn. */ export interface FailConversationTurnInput { conversationId: string; createdAtMs: number; eventId?: string; failureCode: ConversationTurnFailureCode; + failureReason?: ConversationTurnFailureReason; turnId: string; } @@ -71,7 +73,7 @@ export class ConversationTurnLifecycleService implements ConversationTurnLifecyc ]); } - /** Record a classified failure once without accepting raw error details. */ + /** Record a classified failure once. Do not store raw error text. */ async fail(input: FailConversationTurnInput): Promise { await this.events.append(input.conversationId, [ { @@ -81,6 +83,9 @@ export class ConversationTurnLifecycleService implements ConversationTurnLifecyc type: "turn_failed", turnId: input.turnId, failureCode: input.failureCode, + ...(input.failureReason + ? { failureReason: input.failureReason } + : undefined), ...(input.eventId ? { eventId: input.eventId } : undefined), }, }, diff --git a/packages/junior/src/chat/local/runner.ts b/packages/junior/src/chat/local/runner.ts index a44b89a45..41560e8b5 100644 --- a/packages/junior/src/chat/local/runner.ts +++ b/packages/junior/src/chat/local/runner.ts @@ -49,7 +49,10 @@ import { import { credentialContextForActor } from "@/chat/credentials/context"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; -import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; +import type { + ConversationTurnFailureCode, + ConversationTurnFailureReason, +} from "@/chat/conversations/history"; import { persistConversationMessages } from "@/chat/conversations/messages"; import { persistWithRetry } from "@/chat/services/persist-retry"; import { completeAuthPauseTurn } from "@/chat/runtime/auth-pause-state"; @@ -245,6 +248,7 @@ async function runLocalAgentTurnInContext( let completedState: ReturnType; let failureCode: ConversationTurnFailureCode = "persistence_failed"; let modelFailureEventId: string | undefined; + let modelFailureReason: ConversationTurnFailureReason | undefined; let modelFailureCaptureAttempted = false; let currentRunId: string | undefined; let completionSliceId = 1; @@ -422,6 +426,7 @@ async function runLocalAgentTurnInContext( }); reply = finalized.reply; modelFailureEventId = finalized.eventId; + modelFailureReason = finalized.failureReason; if (reply.diagnostics.outcome !== "success") { await deliverAssistantMessage(reply.text); @@ -542,6 +547,7 @@ async function runLocalAgentTurnInContext( createdAtMs: Date.now(), ...(modelFailureEventId ? { eventId: modelFailureEventId } : undefined), failureCode: "model_execution_failed", + ...(modelFailureReason ? { failureReason: modelFailureReason } : undefined), turnId, }); } diff --git a/packages/junior/src/chat/providers/slack/resume.ts b/packages/junior/src/chat/providers/slack/resume.ts index e57b4c934..696617802 100644 --- a/packages/junior/src/chat/providers/slack/resume.ts +++ b/packages/junior/src/chat/providers/slack/resume.ts @@ -750,6 +750,9 @@ async function resumeSlackTurnInContext( outcome: "failed", failureCode: "model_execution_failed", ...(finalized.eventId ? { eventId: finalized.eventId } : undefined), + ...(finalized.failureReason + ? { failureReason: finalized.failureReason } + : undefined), }; }, replyTimeoutMs, diff --git a/packages/junior/src/chat/providers/slack/turn.ts b/packages/junior/src/chat/providers/slack/turn.ts index 93fb55801..d482a82ff 100644 --- a/packages/junior/src/chat/providers/slack/turn.ts +++ b/packages/junior/src/chat/providers/slack/turn.ts @@ -114,6 +114,7 @@ import { } from "@/chat/runtime/turn"; import { buildDeterministicTurnId } from "@/chat/runtime/turn"; import { buildDeterministicAssistantMessageId } from "@/chat/state/turn-id"; +import type { ConversationTurnFailureReason } from "@/chat/conversations/history"; import { markTurnClosed, markTurnFailed } from "@/chat/runtime/turn"; import { startActiveTurn } from "@/chat/runtime/turn"; import { @@ -1170,6 +1171,7 @@ export function createSlackTurn(deps: SlackTurnDeps) { getAgentTurnDiagnosticsAttributes(finalResult); setSpanAttributes(diagnosticsAttributes); let failureEventId: string | undefined; + let failureReason: ConversationTurnFailureReason | undefined; if (finalResult.diagnostics.outcome !== "success") { const finalized = finalizeFailedTurnReplyWithEvent({ reply: finalResult, @@ -1177,6 +1179,7 @@ export function createSlackTurn(deps: SlackTurnDeps) { }); finalResult = finalized.reply; failureEventId = finalized.eventId; + failureReason = finalized.failureReason; await deliverAssistantMessage(finalResult.text); } const turnResult: DispatchTurnResult = @@ -1298,6 +1301,7 @@ export function createSlackTurn(deps: SlackTurnDeps) { return { ...(failureEventId ? { eventId: failureEventId } : undefined), failureCode: "model_execution_failed" as const, + ...(failureReason ? { failureReason } : undefined), outcome: "failed" as const, }; }; diff --git a/packages/junior/src/chat/runtime/turn-execution.ts b/packages/junior/src/chat/runtime/turn-execution.ts index a615d65b2..bafe70e44 100644 --- a/packages/junior/src/chat/runtime/turn-execution.ts +++ b/packages/junior/src/chat/runtime/turn-execution.ts @@ -21,6 +21,7 @@ type SavedTurnResult = eventId?: string; finishedAtMs?: number; failureCode: FailConversationTurnInput["failureCode"]; + failureReason?: FailConversationTurnInput["failureReason"]; outcome: "failed"; }; @@ -75,6 +76,9 @@ export async function executeTurn( ...common, ...(saved.eventId ? { eventId: saved.eventId } : undefined), failureCode: saved.failureCode, + ...(saved.failureReason + ? { failureReason: saved.failureReason } + : undefined), }); } else { await lifecycle.complete({ diff --git a/packages/junior/src/chat/services/turn-failure-response.ts b/packages/junior/src/chat/services/turn-failure-response.ts index fa40a29aa..467c402fb 100644 --- a/packages/junior/src/chat/services/turn-failure-response.ts +++ b/packages/junior/src/chat/services/turn-failure-response.ts @@ -1,3 +1,4 @@ +import type { ConversationTurnFailureReason } from "@/chat/conversations/history"; import { buildTurnFailureResponse } from "@/chat/logging"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import { @@ -45,6 +46,28 @@ function getExecutionFailureReason(reply: { return "empty assistant turn"; } +/** Return a fixed reason label for a failed model turn. */ +export function getTurnFailureReason( + reply: AgentRunResult, +): ConversationTurnFailureReason | undefined { + if (reply.diagnostics.outcome === "success") { + return undefined; + } + if (reply.diagnostics.outcome === "provider_error") { + return findProviderError(reply.diagnostics.providerError)?.kind ?? "unknown"; + } + if (reply.diagnostics.toolErrorCount > 0) { + return "tool_errors"; + } + if ( + reply.diagnostics.usedPrimaryText && + reply.diagnostics.assistantMessageCount > 0 + ) { + return "suppressed_output"; + } + return "empty_output"; +} + function getFailureCapture(reply: AgentRunResult): { attributes: Record; body: string; @@ -110,6 +133,7 @@ export function getAgentTurnDiagnosticsAttributes( /** Sanitized failure fallback plus its optional captured event ID. */ export interface FinalizedTurnFailure { eventId?: string; + failureReason?: ConversationTurnFailureReason; reply: AgentRunResult; } @@ -123,12 +147,16 @@ export function finalizeFailedTurnReplyWithEvent(args: { return { reply: args.reply }; } + const failureReason = getTurnFailureReason(args.reply); const capture = getFailureCapture(args.reply); const eventId = requireTurnFailureEventId( args.logException(capture.error, capture.eventName, { ...getAgentTurnDiagnosticsAttributes(args.reply), ...args.attributes, ...capture.attributes, + ...(failureReason + ? { "app.ai.failure_reason": failureReason } + : undefined), }), capture.eventName, ); @@ -152,6 +180,7 @@ export function finalizeFailedTurnReplyWithEvent(args: { return { eventId, + ...(failureReason ? { failureReason } : undefined), reply: { ...args.reply, text: providerPartialText diff --git a/packages/junior/tests/unit/api/conversation-events.test.ts b/packages/junior/tests/unit/api/conversation-events.test.ts index 0bb369d3a..b1a060e3c 100644 --- a/packages/junior/tests/unit/api/conversation-events.test.ts +++ b/packages/junior/tests/unit/api/conversation-events.test.ts @@ -833,6 +833,7 @@ describe("conversation report event projection", () => { type: "turn_failed", turnId: "turn-1", failureCode: "model_execution_failed", + failureReason: "network", eventId, }), event(5, { @@ -890,13 +891,14 @@ describe("conversation report event projection", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", + failureReason: "network", }); expect(projected[4]?.data).toEqual({ type: "turn_lifecycle", turnId: "turn-delivery-1", state: "failed", - failureKind: "delivery", + failureCode: "delivery_failed", }); const serialized = JSON.stringify(projected); for (const forbidden of [ @@ -909,13 +911,11 @@ describe("conversation report event projection", () => { "private-authorization-id", "private tool result", "private provider error", - "model_execution_failed", eventId, "private-provider", "actorId", "authorizationId", "eventId", - "failureCode", "args", "content", "meta", @@ -1152,7 +1152,7 @@ describe("conversation report event projection", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "delivery", + failureCode: "delivery_failed", }, { type: "tool_calls", @@ -1216,7 +1216,7 @@ describe("conversation report event projection", () => { type: "turn_lifecycle", turnId: "turn-1", state: "failed", - failureKind: "agent", + failureCode: "model_execution_failed", }, }; @@ -1234,7 +1234,7 @@ describe("conversation report event projection", () => { type: "turn_lifecycle", turnId: "turn-1", state: "succeeded", - failureKind: "agent", + failureCode: "model_execution_failed", }, }).success, ).toBe(false); @@ -1253,7 +1253,7 @@ describe("conversation report event projection", () => { expect( conversationReportEventSchema.safeParse({ ...valid, - data: { ...valid.data, failureCode: "private-failure-code" }, + data: { ...valid.data, failureCode: "not_a_real_code" }, }).success, ).toBe(false); expect( diff --git a/packages/junior/tests/unit/services/turn-failure-response.test.ts b/packages/junior/tests/unit/services/turn-failure-response.test.ts index 84b5f9847..4346fab2a 100644 --- a/packages/junior/tests/unit/services/turn-failure-response.test.ts +++ b/packages/junior/tests/unit/services/turn-failure-response.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { getInterruptionMarker } from "@/chat/interruption-marker"; import { createProviderError } from "@/chat/services/provider-error"; -import { finalizeFailedTurnReply } from "@/chat/services/turn-failure-response"; +import { + finalizeFailedTurnReply, + finalizeFailedTurnReplyWithEvent, + getTurnFailureReason, +} from "@/chat/services/turn-failure-response"; import type { AgentRunResult } from "@/chat/services/turn-result"; function providerErrorReply(args: { @@ -52,7 +56,7 @@ describe("finalizeFailedTurnReply", () => { { modelId: "xai/grok-4.5" }, ); - finalizeFailedTurnReply({ + const finalized = finalizeFailedTurnReplyWithEvent({ reply: providerErrorReply({ assistantMessageCount: 0, errorMessage: providerError.message, @@ -62,16 +66,35 @@ describe("finalizeFailedTurnReply", () => { logException, }); + expect(finalized.failureReason).toBe("server"); const attributes = logException.mock.calls[0]?.[2]; expect(attributes).toMatchObject({ "app.ai.provider_error.kind": "server", "app.ai.provider_error.retryable": true, "app.ai.provider_error.status": 503, + "app.ai.failure_reason": "server", "gen_ai.request.model": "xai/grok-4.5", }); expect(attributes).not.toHaveProperty("exception.message"); }); + it("classifies empty execution failures without raw exception text", () => { + const reply: AgentRunResult = { + text: "", + diagnostics: { + outcome: "execution_failure", + modelId: "test-model", + assistantMessageCount: 0, + toolCalls: [], + toolResultCount: 0, + toolErrorCount: 0, + usedPrimaryText: false, + }, + }; + + expect(getTurnFailureReason(reply)).toBe("empty_output"); + }); + it("records a provider failure preserved inside a domain error", () => { const logException = vi.fn().mockReturnValue("evt_guardian"); const providerError = createProviderError("No object generated", {