From 813586384a5818a2e2c893155bb5d382a44875ec Mon Sep 17 00:00:00 2001 From: wenkaifan0720 Date: Wed, 5 Aug 2026 17:46:17 -0700 Subject: [PATCH] osr: expose present liveness, and stop a renderer crash-loop from wedging silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same problem — a frozen tile that nothing can see or recover. sessionStats(): pixel liveness had no observable signal at all. A consumer could prove the renderer executed JavaScript while the texture had been stale for minutes. CefWebSession now counts presents where they already arrive, so this adds no host IPC and no protocol change, and reports presentCount, lastPresentAgoMs, firstPresentSeen and frozen. `frozen` matters: a discarded session produces no presents BY DESIGN, and that must stay distinguishable from a wedge. Renderer crash-loop suicide: a renderer that dies is normally recoverable — OnRenderProcessTerminated reloads and the fresh child takes over. But if the child can no longer be SPAWNED, that reload re-crashes instantly and loops forever while the host notices nothing: the browser process is healthy, the IPC pipe stays up, no processGone is emitted, and the embedder sees a tile stuck on its last frame with no signal and no way back. The observed trigger is the app bundle being replaced under a running host (any rebuild/relaunch), after which every child SIGTRAPs at startup resolving the framework it was launched from — two such crash reports on the reporting machine in one morning. So make the unrecoverable case look like the recoverable one: past a burst threshold (4 deaths in 10s, unreachable by ordinary flakiness), exit deliberately. The pipe EOFs, the plugin reports processGone, and the embedder's existing recreate funnel spawns a fresh host — which re-resolves the binary from disk and so picks up the NEW bundle. A silent wedge becomes the recovery path that already works. Deliberately NOT changed: the steady-state liveness sweep still accepts a post-establishment no-present browser as healthy-static. Its audited comment is right that a nudge cannot distinguish "nothing to paint" from "can't paint", and naive escalation previously recreate-stormed every static tile. sessionStats gives consumers the data to make that call with better context than the sweep has. --- lib/flutter_cef.dart | 1 + lib/src/cef_web_controller.dart | 19 +++++++ .../macos/Classes/CefWebSession.swift | 32 +++++++++++ .../macos/Classes/FlutterCefPlugin.swift | 15 +++++ .../flutter_cef_macos/native/cef_host/main.mm | 55 +++++++++++++++++++ .../lib/src/cef_events.dart | 40 ++++++++++++++ 6 files changed, 162 insertions(+) diff --git a/lib/flutter_cef.dart b/lib/flutter_cef.dart index d2eb982..d7495a5 100644 --- a/lib/flutter_cef.dart +++ b/lib/flutter_cef.dart @@ -15,6 +15,7 @@ export 'package:flutter_cef_platform_interface/flutter_cef_platform_interface.da CefFindResult, CefJsDialogRequest, CefLoadError, + CefSessionStats, CefMediaPermissionRequest, CefMediaSetting, CefMediaState, diff --git a/lib/src/cef_web_controller.dart b/lib/src/cef_web_controller.dart index 402a416..680c55f 100644 --- a/lib/src/cef_web_controller.dart +++ b/lib/src/cef_web_controller.dart @@ -975,6 +975,25 @@ class CefWebController { } + /// Read this session's pixel-liveness counters (see [CefSessionStats]). + /// + /// Returns null when the platform has no such session (never created, or + /// already disposed). Cheap — the counters are maintained where present + /// frames already arrive, so this adds no IPC to the host. + Future sessionStats() async { + final raw = await _channel.invokeMapMethod( + 'sessionStats', + {'sessionId': sessionId}, + ); + if (raw == null) return null; + return CefSessionStats( + presentCount: (raw['presentCount'] as num?)?.toInt() ?? 0, + lastPresentAgoMs: (raw['lastPresentAgoMs'] as num?)?.toInt(), + firstPresentSeen: raw['firstPresentSeen'] as bool? ?? false, + frozen: raw['frozen'] as bool? ?? false, + ); + } + /// Mute or unmute the page's audio output. Besides silencing it, a hidden /// AND muted page regains Chromium's intensive wake-up throttling (audible /// pages are exempt), so muting on hide keeps a background tile's timers diff --git a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift index be294fc..c1f4c83 100644 --- a/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift +++ b/packages/flutter_cef_macos/macos/Classes/CefWebSession.swift @@ -117,6 +117,35 @@ final class CefWebSession: NSObject, FlutterTexture { let sessionId: String private(set) var textureId: Int64 = 0 + // ── Present liveness counters ──────────────────────────────────────────── + // Pixel liveness had NO observable signal: a probe could prove the renderer's + // JS loop was alive (eval round-trip) while the texture had been frozen for + // minutes — the exact "JS alive, pixels dead" wedge. These count what actually + // reaches the texture, so a consumer can tell a live page from a stale one + // without guessing. Written on the reader thread, read from the main thread + // via sessionStats(), so both go through presentStatsLock. + private let presentStatsLock = NSLock() + private var _presentCount: UInt64 = 0 + private var _lastPresentUptimeNs: UInt64 = 0 + + /// Total presents this session has delivered, the wall-clock age of the most + /// recent one (nil if none), and whether it ever painted. + func presentStats() -> (count: UInt64, lastAgoMs: Int?, firstSeen: Bool) { + presentStatsLock.lock() + defer { presentStatsLock.unlock() } + let ago: Int? = _lastPresentUptimeNs == 0 + ? nil + : Int((DispatchTime.now().uptimeNanoseconds &- _lastPresentUptimeNs) / 1_000_000) + return (_presentCount, ago, _presentCount > 0) + } + + private func notePresent() { + presentStatsLock.lock() + _presentCount &+= 1 + _lastPresentUptimeNs = DispatchTime.now().uptimeNanoseconds + presentStatsLock.unlock() + } + // Wire binding to the owning host: the host this session is multiplexed on and // the Swift-assigned browserId it routes by. Set once via attach() right after // CefProfileHost.createBrowser() allocates the id. @@ -649,6 +678,9 @@ final class CefWebSession: NSObject, FlutterTexture { func handleFrame(_ op: UInt8, _ payload: [UInt8]) { switch op { case Self.opPresent: + // Count FIRST: a present that arrives but fails to adopt is still proof + // the producer is painting, which is what liveness asks about. + notePresent() // Read textureId under bufferLock — dispose() writes it under the same // lock on the main thread, so this avoids a data race on the Int64. bufferLock.lock() diff --git a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift index 74cff0b..64aea41 100644 --- a/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift +++ b/packages/flutter_cef_macos/macos/Classes/FlutterCefPlugin.swift @@ -125,6 +125,21 @@ public class FlutterCefPlugin: NSObject, FlutterPlugin { case "loadTrusted": loadTrusted(args, result) case "resize": resize(args, result) case "getFrameSurface": getFrameSurface(args, result) + case "sessionStats": + // Pixel liveness, for a consumer that needs to tell a live page from a + // stale texture. A JS round-trip proves only that the renderer executes + // script; presentCount/lastPresentAgoMs prove frames are still reaching + // the texture, which is what "frozen" actually means to a user. + guard let sid = args["sessionId"] as? String, let s = sessions[sid] else { + result(nil); return + } + let st = s.presentStats() + result([ + "presentCount": Int(clamping: st.count), + "lastPresentAgoMs": st.lastAgoMs as Any, + "firstPresentSeen": st.firstSeen, + "frozen": frozenSessions.contains(sid), + ]) case "dispose": destroy(args, result) case "pointer": pointer(args, result) case "key": key(args, result) diff --git a/packages/flutter_cef_macos/native/cef_host/main.mm b/packages/flutter_cef_macos/native/cef_host/main.mm index c501cfc..61b97fe 100644 --- a/packages/flutter_cef_macos/native/cef_host/main.mm +++ b/packages/flutter_cef_macos/native/cef_host/main.mm @@ -50,6 +50,7 @@ #include #include +#include #include #include #include @@ -177,6 +178,50 @@ // the SendFrame `< 0` check against the teardown `= -1` store (UB, and benign // only because a closed-fd write is a safe no-op); the atomic makes it defined. std::atomic g_ipc_fd{-1}; + +// ── Renderer crash-loop detector ───────────────────────────────────────────── +// A renderer that dies is normally recoverable: OnRenderProcessTerminated +// reloads and the fresh child takes over. But if the child can no longer be +// SPAWNED at all, that reload re-crashes instantly and loops forever, and the +// host never notices: the browser process is healthy, so the IPC pipe stays up, +// no processGone is ever emitted, and the embedder sees a tile stuck on its last +// frame with no signal and no way back. The observed trigger is the app bundle +// being replaced under a running host (a rebuild/relaunch): every child then +// SIGTRAPs at startup resolving the CEF framework it was launched from. +// +// So make the unrecoverable case LOOK like the recoverable one. Past a burst +// threshold, exit deliberately: the pipe EOFs, the plugin reports processGone, +// and the embedder's existing recreate funnel spawns a fresh host — which +// re-resolves the binary from disk and therefore picks up the NEW bundle. A +// silent wedge becomes the recovery path that already works. +std::mutex g_renderer_crash_mutex; +int g_renderer_crash_count = 0; +std::chrono::steady_clock::time_point g_renderer_crash_window_start; +// Tuned to be unreachable by ordinary flakiness: an isolated renderer crash (or +// a few across unrelated tabs) reloads normally and never trips this. +constexpr int kRendererCrashBurstLimit = 4; +constexpr std::chrono::seconds kRendererCrashWindow{10}; + +void DoShutdown(); // defined below; the crash-loop exit reuses it + +/// Record a renderer death and report whether they are arriving as a burst — +/// i.e. the reload is re-crashing rather than recovering. UI-thread only (the +/// single caller is OnRenderProcessTerminated), but locked anyway since the +/// counter is process-global across every slot: a stale-bundle host fails for +/// ALL its browsers at once, which is exactly the signal we want to add up. +bool NoteRendererCrashAndCheckLoop() { + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(g_renderer_crash_mutex); + if (g_renderer_crash_count == 0 || + now - g_renderer_crash_window_start > kRendererCrashWindow) { + // First death, or the previous burst aged out: start a fresh window so + // unrelated one-off crashes over a long session never accumulate. + g_renderer_crash_window_start = now; + g_renderer_crash_count = 1; + return false; + } + return ++g_renderer_crash_count >= kRendererCrashBurstLimit; +} std::mutex g_ipc_write_mutex; // Per-browser state. One cef_host process now multiplexes N browsers (one per @@ -1519,6 +1564,16 @@ void OnRenderProcessTerminated(CefRefPtr browser, SendLog(slot_->browser_id, "renderer terminated (status " + std::to_string(status) + ") — reloading"); if (router_) router_->OnRenderProcessTerminated(browser); + if (NoteRendererCrashAndCheckLoop()) { + // Reloading again would just re-crash: the children can't start. Exit so + // the embedder's processGone → recreate path takes over (see the detector). + SendLog(0, + "renderer crash LOOP (" + std::to_string(kRendererCrashBurstLimit) + + " in " + std::to_string(kRendererCrashWindow.count()) + + "s) — children cannot start; exiting so the host is respawned"); + DoShutdown(); + return; + } if (browser) browser->ReloadIgnoreCache(); } diff --git a/packages/flutter_cef_platform_interface/lib/src/cef_events.dart b/packages/flutter_cef_platform_interface/lib/src/cef_events.dart index 06a8451..54e5af6 100644 --- a/packages/flutter_cef_platform_interface/lib/src/cef_events.dart +++ b/packages/flutter_cef_platform_interface/lib/src/cef_events.dart @@ -153,6 +153,46 @@ class CefMediaState { 'CefMediaState(video: $videoActive, audio: $audioActive, $setting)'; } +/// Pixel-liveness counters for a session: how many frames have actually +/// reached the texture, and how long ago the last one did. +/// +/// The signal that was missing. A JavaScript round-trip proves the renderer +/// still executes script, which a page can do for minutes after compositing +/// has died — the "JS alive, pixels dead" wedge. [presentCount] advancing is +/// the only honest evidence the page is still being drawn. +class CefSessionStats { + const CefSessionStats({ + required this.presentCount, + required this.lastPresentAgoMs, + required this.firstPresentSeen, + required this.frozen, + }); + + /// Frames delivered to the texture since this session was created. Compare + /// two readings over time: unchanged while the page should be animating + /// means the pixels are wedged, whatever a JS probe says. + final int presentCount; + + /// Milliseconds since the most recent present, or null if none has arrived. + /// High on a genuinely static page too — pair it with [presentCount] deltas + /// rather than treating it as a fault on its own. + final int? lastPresentAgoMs; + + /// Whether this session has ever painted. False means establishment never + /// completed; the first-paint watchdog owns that case. + final bool firstPresentSeen; + + /// Whether the native browser is currently discarded (the texture is holding + /// the last painted frame on purpose). A frozen session produces no presents + /// BY DESIGN, so this distinguishes intended stillness from a wedge. + final bool frozen; + + @override + String toString() => + 'CefSessionStats(presents: $presentCount, lastAgoMs: $lastPresentAgoMs, ' + 'painted: $firstPresentSeen, frozen: $frozen)'; +} + /// The live frame surface backing a session: the global IOSurface id its /// off-screen CVPixelBuffer is wrapped over, plus the surface's PHYSICAL /// (Retina) pixel dimensions. Delivered by [CefWebController.onSurface] on each