From 29ba9a8c61f128a99f804dd404bcb29f20a31637 Mon Sep 17 00:00:00 2001 From: BB-fat <1056871944@qq.com> Date: Wed, 12 Aug 2026 11:29:40 +0000 Subject: [PATCH] fix(screenshot): fall back to CDP capture when captureVisibleTab fails chrome.tabs.captureVisibleTab reads back the window surface, which fails outright on some Windows/Chrome combinations (Chromium FAILURE_REASON_READBACK_FAILED: "Failed to capture tab: image readback failed"), leaving full-tab screenshots without any working path. Capture the full tab via CDP Page.captureScreenshot (fromSurface) when the visible-tab capture rejects; the CDP path goes through the renderer BeginFrame pipeline instead of the surface readback. When both paths fail, return cdp_failed with data.reason=screenshot_capture_failed carrying both underlying messages, and render a CLI hint that points at the browser-side readback cause. Refs #71 --- .../src/tools/__tests__/observation.test.ts | 65 ++++++++++++++ apps/extension/src/tools/observation.ts | 90 ++++++++++++++----- apps/extension/src/transport/types.ts | 3 +- crates/bsk-cli/src/cli/render_error.rs | 22 +++++ 4 files changed, 157 insertions(+), 23 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 519d7db..bfbcb2f 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -196,6 +196,71 @@ describe("handleScreenshot", () => { expect(res).toMatchObject({ code: "cdp_failed", message: /captureVisibleTab refused/ }); }); + it("falls back to CDP Page.captureScreenshot when captureVisibleTab fails", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const capture = vi.fn(async () => { + throw new Error("Failed to capture tab: image readback failed"); + }); + const { cdp, sent } = makeFakeCdp({ + "Page.captureScreenshot": () => ({ data: TINY_PNG }), + }); + const res = await handleScreenshot( + sm, + { session_id: "aa11" }, + makeScreenshotDeps({ cdp, captureVisibleTab: capture }), + ); + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.tab_id).toBe(7); + expect(res.image_base64).toBe(TINY_PNG); + expect(res.format).toBe("png"); + expect(capture).toHaveBeenCalledTimes(1); + const fallbackCall = sent.find((c) => c.method === "Page.captureScreenshot"); + expect(fallbackCall?.params).toEqual({ format: "png", fromSurface: true }); + }); + + it("reports screenshot_capture_failed when both capture paths fail", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const capture = vi.fn(async () => { + throw new Error("Failed to capture tab: image readback failed"); + }); + const { cdp } = makeFakeCdp({ + "Page.captureScreenshot": () => { + throw new Error("debugger detached"); + }, + }); + const res = await handleScreenshot( + sm, + { session_id: "aa11" }, + makeScreenshotDeps({ cdp, captureVisibleTab: capture }), + ); + expect(res).toMatchObject({ + code: "cdp_failed", + data: { reason: "screenshot_capture_failed" }, + }); + const message = (res as { message?: string }).message ?? ""; + expect(message).toContain("image readback failed"); + expect(message).toContain("debugger detached"); + }); + + it("falls back to CDP when captureVisibleTab returns and CDP yields data", async () => { + // Sanity: a successful primary path never touches CDP even when a + // runner is available. + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const capture = vi.fn(async () => `data:image/png;base64,${TINY_PNG}`); + const { cdp, sent } = makeFakeCdp({}); + const res = await handleScreenshot( + sm, + { session_id: "aa11" }, + makeScreenshotDeps({ cdp, captureVisibleTab: capture }), + ); + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.image_base64).toBe(TINY_PNG); + expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); + }); + it("captures a clipped PNG when ref is given", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index a075dbb..34c60f1 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -17,7 +17,7 @@ import { type CaptureSuppressSendToTab, withOverlaysHiddenForCapture, } from "@/lib/capture-suppress-bridge"; -import type { SessionManager } from "@/session-manager/manager"; +import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { GetHtmlParams, GetHtmlResult, @@ -37,6 +37,7 @@ import { enforceToolTargetScope, isRpcError, lookupSession, + type ResolvedTargetTab, resolveCdpAccessibleTargetTab, type CdpRunner as SharedCdpRunner, normaliseRef as sharedNormaliseRef, @@ -175,6 +176,57 @@ async function captureElementScreenshot( } } +/** + * Full-tab PNG capture. The primary path is `chrome.tabs.captureVisibleTab`; + * when it rejects, fall back to CDP `Page.captureScreenshot` with + * `fromSurface: true`. `captureVisibleTab` reads back the window surface, + * which fails outright on some Windows/Chrome combinations (Chromium's + * FAILURE_REASON_READBACK_FAILED — "Failed to capture tab: image readback + * failed"); the CDP path captures through the renderer's BeginFrame + * pipeline instead, which does not depend on that readback. + * + * When both paths fail the returned error carries + * `data.reason = "screenshot_capture_failed"` and both underlying messages + * so the CLI can point the user at the browser-side cause. + */ +async function captureFullTabPng( + deps: ScreenshotDeps, + ctx: SessionContext, + target: ResolvedTargetTab, +): Promise { + try { + const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }); + return stripDataUrlPrefix(dataUrl); + } catch (primaryErr) { + const cdp = deps.cdp; + if (!cdp) { + return { + code: "cdp_failed", + message: primaryErr instanceof Error ? primaryErr.message : String(primaryErr), + }; + } + let fallbackMsg: string; + try { + cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + await cdp.ensureAttachedToUrl?.(target.tabId, target.url); + const shot = await cdp.send<{ data?: string }>(target.tabId, "Page.captureScreenshot", { + format: "png", + fromSurface: true, + }); + if (shot.data) return shot.data; + fallbackMsg = "Page.captureScreenshot returned no data"; + } catch (fallbackErr) { + fallbackMsg = fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr); + } + const primaryMsg = primaryErr instanceof Error ? primaryErr.message : String(primaryErr); + return rpcError( + "cdp_failed", + "screenshot_capture_failed", + `captureVisibleTab failed: ${primaryMsg}; CDP Page.captureScreenshot fallback failed: ${fallbackMsg}`, + ); + } +} + export async function handleScreenshot( manager: SessionManager, params: ScreenshotParams, @@ -228,27 +280,21 @@ export async function handleScreenshot( ); } - try { - const dataUrl = await withOverlaysHiddenForCapture( - target.tabId, - () => deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }), - deps.sendToTab, - ); - const image_base64 = stripDataUrlPrefix(dataUrl); - const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 }; - return withShotDialogs({ - image_base64, - width: dims.width, - height: dims.height, - format: "png", - tab_id: target.tabId, - }); - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } + const captured = await withOverlaysHiddenForCapture( + target.tabId, + () => captureFullTabPng(deps, ctx, target), + deps.sendToTab, + ); + if (isRpcError(captured)) return captured; + const image_base64 = captured; + const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 }; + return withShotDialogs({ + image_base64, + width: dims.width, + height: dims.height, + format: "png", + tab_id: target.tabId, + }); } // --------------------------------------------------------------------------- diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 4533d7d..47de957 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -31,7 +31,8 @@ export type RpcErrorReason = | "single_select_value_count" | "tab_not_active" | "restricted_tab_url" - | "borrow_conflict"; + | "borrow_conflict" + | "screenshot_capture_failed"; export interface RpcErrorData { reason?: RpcErrorReason; diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 30cd5bb..415387e 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -48,6 +48,7 @@ pub mod reason { pub const SINGLE_SELECT_VALUE_COUNT: &str = "single_select_value_count"; pub const TAB_NOT_ACTIVE: &str = "tab_not_active"; pub const BORROW_CONFLICT: &str = "borrow_conflict"; + pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed"; pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY; } @@ -261,6 +262,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, + (ErrorCode::CdpFailed, reason::SCREENSHOT_CAPTURE_FAILED) => RenderInfo { + summary: "the browser could not capture the tab image", + hint: Some( + "both the visible-tab capture and the CDP compositor fallback failed inside the browser; this points at a browser-side rendering/readback issue — try reloading the tab, restarting the browser, or upgrading/downgrading Chrome", + ), + exit_code: base.exit_code, + }, _ => base, } } @@ -370,6 +378,20 @@ mod tests { assert_eq!(info.exit_code, 4); } + #[test] + fn screenshot_capture_failed_overrides_cdp_failed_copy() { + let data = serde_json::json!({ "reason": reason::SCREENSHOT_CAPTURE_FAILED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&data)); + assert_eq!(info.summary, "the browser could not capture the tab image"); + assert!( + info.hint.unwrap().contains("CDP compositor fallback"), + "expected screenshot-specific hint, got {:?}", + info.hint + ); + // Still the browser/CDP failure bucket. + assert_eq!(info.exit_code, 3); + } + #[test] fn ref_not_found_overrides_not_found_copy() { let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });