Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions tests/web/app-render.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,47 @@ it("follows same-key streamed growth, preserves reading position, and honors exp
expect(scroll).toHaveBeenCalledOnce();
});

it("renders persisted provider failures with partial output and a bounded cause", () => {
const snapshot = activeSnapshot();
snapshot.selectedSession!.entries = [
{
type: "message",
id: "user",
timestamp: "2026-09-13T00:00:00Z",
message: { role: "user", content: "run" },
},
{
type: "message",
id: "assistant-error",
timestamp: "2026-09-13T00:00:01Z",
message: {
role: "assistant",
content: "partial output",
parts: [{ type: "text", text: "partial output" }],
stopReason: "error",
errorMessage: "Synthetic provider failure",
},
},
];
renderWithI18n(
createElement(Transcript, {
snapshot,
liveMessages: [],
liveRunning: false,
livePhase: "idle",
liveRetry: null,
thinkingStarts: {},
thinkingDurations: {},
scrollToBottom: 0,
onResend: async () => true,
}),
);

expect(screen.getByText("partial output")).toBeTruthy();
expect(screen.getByText(i18n.t("assistantFailed"))).toBeTruthy();
expect(screen.getByText("Synthetic provider failure")).toBeTruthy();
});

it("shows cancellation and queued follow-up receipts on the active session", () => {
const snapshot = activeSnapshot();
const store = createWebStore();
Expand Down Expand Up @@ -428,6 +469,29 @@ it("shows cancellation and queued follow-up receipts on the active session", ()
).toBe(true);
});

it("keeps failed and uncertain turn outcomes distinct in the composer hint", () => {
const snapshot = activeSnapshot();
const base = thinkingProps(snapshot);
const { rerender } = renderWithI18n(
createElement(Composer, {
...base,
turnTerminalStatus: "failed",
}),
);
expect(screen.getByText(i18n.t("failedTurn"))).toBeTruthy();
rerender(
createElement(
I18nextProvider,
{ i18n },
createElement(Composer, {
...base,
turnTerminalStatus: "uncertain",
}),
),
);
expect(screen.getByText(i18n.t("uncertainTurn"))).toBeTruthy();
});

it("keeps background terminal activity and omission receipts visible", () => {
const snapshot = activeSnapshot();
snapshot.runtime.capabilities = {
Expand Down
111 changes: 111 additions & 0 deletions tests/web/openpi-web-provider.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,114 @@ test("thinking level reaches the provider request end to end", async ({
await rm(workspace, { recursive: true, force: true });
}
});

test("provider failures remain visible through SSE, snapshot, and reload", async ({
page,
}, testInfo) => {
const provider = await startFakeProvider();
const workspace = await mkdtemp(join(tmpdir(), "openpi-provider-error-"));
try {
const imported = await page.request.post("/api/workspaces", {
headers: authHeaders,
data: { path: workspace },
});
expect(imported.status()).toBe(201);
const { path: canonicalWorkspace } = await imported.json();
const created = await page.request.post("/api/sessions", {
headers: authHeaders,
data: {
workspacePath: canonicalWorkspace,
commandId: "provider-error-e2e-session",
},
});
expect(created.status()).toBe(201);

const initialSnapshot = await page.request.get("/api/snapshot", {
headers: authHeaders,
});
const sessionId = (await initialSnapshot.json()).currentSessionId as string;
const selected = await page.request.post("/api/model", {
headers: authHeaders,
data: { provider: PROVIDER_ID, modelId: MODEL_ID, sessionId },
});
expect(selected.status()).toBe(200);

await page.goto("/", { waitUntil: "domcontentloaded" });
const input = page.getByRole("textbox", { name: "描述任务" });
await expect(input).toBeVisible();
provider.failResponses();
await input.fill("Trigger a provider failure");
await page.getByRole("button", { name: "发送", exact: true }).click();

const readSnapshot = async () => {
const response = await page.request.get("/api/snapshot", {
headers: authHeaders,
});
expect(response.status()).toBe(200);
return (await response.json()) as {
runtime?: { status?: string };
selectedSession?: {
entries: Array<{
message?: {
role?: string;
stopReason?: string;
errorMessage?: string;
};
}>;
};
};
};
const finalState = async () => {
const snapshot = await readSnapshot();
const assistants = (snapshot.selectedSession?.entries ?? [])
.map((entry) => entry.message)
.filter((message) => message?.role === "assistant");
const lastAssistant = assistants.at(-1);
return {
runtimeStatus: snapshot.runtime?.status,
stopReason: lastAssistant?.stopReason,
hasErrorMessage:
lastAssistant?.errorMessage?.includes("Synthetic provider failure") ??
false,
};
};
await expect
.poll(finalState, { timeout: 30_000, intervals: [200, 500, 1_000] })
.toEqual({
runtimeStatus: "idle",
stopReason: "error",
hasErrorMessage: true,
});
expect(provider.requests.length).toBeGreaterThan(1);

const failure = page.locator(".assistant-outcome.error").last();
await expect(failure).toContainText("助手回复失败。", { timeout: 30_000 });
await expect(failure).toContainText("Synthetic provider failure", {
timeout: 30_000,
});
await expect(page.locator(".composer-hint")).toContainText(
"当前轮次失败。",
{ timeout: 30_000 },
);
const snapshot = await readSnapshot();
const assistants = (snapshot.selectedSession?.entries ?? [])
.map((entry) => entry.message)
.filter((message) => message?.role === "assistant");
const assistant = assistants.at(-1);
expect(assistant?.stopReason).toBe("error");
expect(assistant?.errorMessage).toContain("Synthetic provider failure");
await page.screenshot({
path: testInfo.outputPath("provider-error-final.png"),
fullPage: true,
});

await page.reload({ waitUntil: "domcontentloaded" });
await expect(page.locator(".assistant-outcome.error").last()).toContainText(
"Synthetic provider failure",
{ timeout: 30_000 },
);
} finally {
await provider.close();
await rm(workspace, { recursive: true, force: true });
}
});
80 changes: 80 additions & 0 deletions tests/web/pi-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1514,6 +1514,86 @@ test("message_end and queued prompts do not settle a running turn", () => {
);
});

test("projects provider failure details through the live message and retry lifecycle", () => {
const session = { sessionManager: { getSessionId: () => "session" } };
const harness = Object.create(PiWebRuntime.prototype) as RuntimeHarness;
harness.runtime = { session };
harness.pendingPromptTraces = [];
harness.liveMessageSequence = 0;
harness.listeners = new Set();
harness.nextTurnEpoch = 0;
harness.terminalTurnKeys = new Set();
harness.turnSettlementWaiters = new Map();
harness.turnAbortOperations = new Map();
const events: WebRuntimeEvent[] = [];
harness.listeners.add((event) => events.push(event));
harness.activePromptTrace = {
commandId: "provider-error",
sessionId: "session",
startedAt: 1,
started: false,
queued: false,
};

const projectEvent = (
PiWebRuntime.prototype as unknown as {
projectEvent(this: RuntimeHarness, session: object, event: object): void;
}
).projectEvent;

projectEvent.call(harness, session, { type: "agent_start" });
projectEvent.call(harness, session, {
type: "message_start",
message: { role: "user", content: [{ type: "text", text: "run" }] },
});
projectEvent.call(harness, session, {
type: "message_end",
message: {
role: "assistant",
content: [{ type: "text", text: "partial output" }],
stopReason: "error",
errorMessage: "Synthetic provider failure",
},
});
projectEvent.call(harness, session, {
type: "auto_retry_start",
attempt: 1,
maxAttempts: 2,
delayMs: 10,
errorMessage: "Retrying provider request",
});
projectEvent.call(harness, session, {
type: "auto_retry_end",
attempt: 2,
success: false,
finalError: "Synthetic provider failure",
});
projectEvent.call(harness, session, { type: "agent_settled" });

const message = events.find((event) => event.type === "message_end")?.detail
?.message as Record<string, unknown> | undefined;
assert.equal(message?.content, "partial output");
assert.equal(message?.stopReason, "error");
assert.equal(message?.errorMessage, "Synthetic provider failure");
assert.deepEqual(
events.find((event) => event.type === "auto_retry_end")?.detail,
{
attempt: 2,
success: false,
finalError: "Synthetic provider failure",
},
);
assert.deepEqual(
events.find((event) => event.type === "turn_settled")?.detail,
{
sessionId: "session",
commandId: "provider-error",
epoch: 1,
outcome: "failed",
},
);
});

test("toolUse message_end without a terminal result settles as uncertain", () => {
const session = { sessionManager: { getSessionId: () => "session" } };
const harness = Object.create(PiWebRuntime.prototype) as RuntimeHarness;
Expand Down
33 changes: 33 additions & 0 deletions tests/web/protocol.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import test from "node:test";
import {
projectEntries,
projectMessage,
WEB_MAX_TEXT,
WEB_MAX_MESSAGE_PARTS,
WEB_MAX_SELECTED_TRANSCRIPT_BYTES,
} from "../../web/protocol/types.ts";
Expand All @@ -19,6 +20,38 @@ test("message projection does not create phantom text for detail-only messages",
assert.equal(projected.parts?.length, 2);
});

test("message projection preserves bounded assistant failure diagnostics", () => {
const projected = projectMessage({
role: "assistant",
content: [{ type: "text", text: "partial output" }],
stopReason: "error",
errorMessage: "Synthetic provider quota failure",
});

assert.equal(projected.content, "partial output");
assert.deepEqual(projected.parts, [{ type: "text", text: "partial output" }]);
assert.equal(projected.stopReason, "error");
assert.equal(projected.errorMessage, "Synthetic provider quota failure");
});

test("message projection bounds and sanitizes provider failure diagnostics", () => {
const projected = projectMessage({
role: "assistant",
content: [],
stopReason: "error",
errorMessage: `provider\u001b[31m failure\u0000 ${"x".repeat(WEB_MAX_TEXT)}`,
});

assert.equal(projected.errorMessage?.includes("\u001b"), false);
assert.equal(projected.errorMessage?.includes("\u0000"), false);
assert.equal(projected.truncation?.truncated, true);
assert.equal(projected.truncation?.text, true);
assert.ok(
(projected.errorMessage?.length ?? 0) <=
WEB_MAX_TEXT + "\n[truncated]".length,
);
});

test("message projection keeps text parts separated without phantom blank lines", () => {
const projected = projectMessage({
role: "assistant",
Expand Down
25 changes: 24 additions & 1 deletion tests/web/provider-e2e-support.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,24 @@ export function seedAgentDirectory(agentDirectory: string) {
// reach the provider, rather than a no-op against the initial clamped level.
writeFileSync(
join(agentDirectory, "settings.json"),
`${JSON.stringify({ defaultThinkingLevel: "off" }, null, 2)}\n`,
`${JSON.stringify(
{
defaultThinkingLevel: "off",
// Keep the provider failure test bounded while still exercising retry.
retry: { enabled: true, maxRetries: 1, baseDelayMs: 10 },
},
null,
2,
)}\n`,
);
}

export type FakeProvider = {
readonly requests: RecordedProviderRequest[];
/** Hold the next response until {@link release} settles it. */
holdNextResponse(): void;
/** Fail provider responses until the server is closed. */
failResponses(errorMessage?: string): void;
/** Release a held response so the turn can settle. */
release(): void;
close(): Promise<void>;
Expand Down Expand Up @@ -114,6 +124,7 @@ export async function startFakeProvider(): Promise<FakeProvider> {
const requests: RecordedProviderRequest[] = [];
let pendingRelease: (() => void) | undefined;
let holdRequested = false;
let failureMessage: string | undefined;

const server: Server = createServer(async (request, response) => {
const path = request.url ?? "/";
Expand All @@ -135,6 +146,15 @@ export async function startFakeProvider(): Promise<FakeProvider> {
headers: request.headers,
body,
});
if (failureMessage !== undefined) {
response.writeHead(500, { "Content-Type": "application/json" });
response.end(
JSON.stringify({
error: { message: failureMessage, type: "provider_error" },
}),
);
return;
}
if (holdRequested) {
holdRequested = false;
await new Promise<void>((resolve) => {
Expand Down Expand Up @@ -163,6 +183,9 @@ export async function startFakeProvider(): Promise<FakeProvider> {
holdNextResponse() {
holdRequested = true;
},
failResponses(errorMessage = "Synthetic provider failure") {
failureMessage = errorMessage;
},
release() {
const release = pendingRelease;
pendingRelease = undefined;
Expand Down
Loading
Loading