From 8f7b73030f35470cbd13a0005fa6b8e875b01572 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 30 Aug 2026 17:19:24 -0400 Subject: [PATCH 1/5] fix(macos): capture exact off-space windows --- CHANGELOG.md | 11 + README.md | 7 +- docs/WINDOW_CAPTURE.md | 14 +- openadapt_capture/control.py | 7 + openadapt_capture/events.py | 2 + openadapt_capture/input_observer/darwin.py | 11 + openadapt_capture/recorder.py | 47 ++- openadapt_capture/video.py | 8 + openadapt_capture/window_capture.py | 381 ++++++++++++++++++--- pyproject.toml | 2 + tests/test_control.py | 4 +- tests/test_input_observer_darwin.py | 15 + tests/test_video.py | 5 + tests/test_window_capture.py | 125 +++++++ uv.lock | 45 +++ 15 files changed, 627 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43b926d..83cff7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ _This release is published under the MIT License._ ### Features +- Capture an exact macOS window through a desktop-independent + ScreenCaptureKit filter. Exact-window Quartz and system-utility providers + remain as compatibility paths. - **capture**: Seal native action geometry at capture time ([#94](https://github.com/OpenAdaptAI/openadapt-capture/pull/94), [`08727ab`](https://github.com/OpenAdaptAI/openadapt-capture/commit/08727ab0e471d4a6de3c460099b1a8a97758cf6f)) @@ -37,6 +40,14 @@ _This release is published under the MIT License._ ### Bug Fixes +- Refuse ambiguous macOS window selectors instead of choosing a different + matching window. +- Keep FFmpeg outside the recorder's interrupt process group so Ctrl-C can + finish the video trailer. +- Ignore unattributable macOS modifier-flag events without terminating the + input observer. +- Report failed and finalizing sessions with `ready: false`, plus a stable + failure stage and error code. - **ci**: Allowlist the published extension key by exact value ([#112](https://github.com/OpenAdaptAI/openadapt-capture/pull/112), [`69a123f`](https://github.com/OpenAdaptAI/openadapt-capture/commit/69a123fccfa6dcbf848e486301832cc826ab679a)) diff --git a/README.md b/README.md index e62ac6e..5ad10c3 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,11 @@ producing media that looks complete but has an evidence gap. The full contract, including the multi-monitor rules and the coordinate-space flags converters must respect, is in [docs/WINDOW_CAPTURE.md](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/WINDOW_CAPTURE.md). +On current macOS, exact-window capture uses ScreenCaptureKit and does not +require the window to be frontmost. It can capture an occluded window or a +window on another Space. The recorder still needs a logged-in desktop session, +and a minimized window must return a valid exact frame. + Linux window mode needs X11 with EWMH and XComposite. It refuses to start under native Wayland or XWayland-only. @@ -263,7 +268,7 @@ Flow instead: [docs/BROWSER_EXTENSION_BOUNDARY.md](https://github.com/OpenAdaptA ## Limits -- Native recording needs a visible user session plus the operating system's +- Native recording needs a logged-in user session plus the operating system's screen-recording and input-monitoring permissions. - Accessibility evidence appears only where the application and the local provider expose it. An opaque remote application still needs Flow's visual diff --git a/docs/WINDOW_CAPTURE.md b/docs/WINDOW_CAPTURE.md index 5c5aeb4..0cc8546 100644 --- a/docs/WINDOW_CAPTURE.md +++ b/docs/WINDOW_CAPTURE.md @@ -37,10 +37,17 @@ matching how `openadapt-flow` identifies the same window at replay time. The selectors can also be set through config or environment (`RECORD_WINDOW_OWNER` / `RECORD_WINDOW_TITLE`). +On macOS, a selector that still matches multiple capturable windows fails. +Use the complete title. Capture never chooses the largest match when that can +bind the recording to a different window. + In this mode: -- **Frames are the target window's pixels.** macOS captures the window's own - buffer (`CGWindowListCreateImage`, the identical call flow's replay uses); +- **Frames are the target window's pixels.** On current macOS, ScreenCaptureKit + captures a desktop-independent exact-window filter. The legacy Quartz image + API and `/usr/sbin/screencapture -o -l` remain exact-window compatibility + paths. This supports an occluded window and a window on another Space. A + minimized window must still return a valid exact frame or the session fails. Linux X11 reads an XComposite named-window pixmap. It doesn't use a root screenshot, so another window cannot replace the target pixels. Windows grabs the window's screen region, so keep the window unoccluded. @@ -72,6 +79,9 @@ In this mode: lost window, capture failure, or unexpected output-frame size fails the session instead of producing complete-looking media with an evidence gap. +The persisted frame state names the provider that produced the pixels and +whether the provider was independent of window visibility. + Linux window mode requires an X11 session with EWMH and XComposite. Capture won't start window mode in a native Wayland or XWayland-only session. A future Wayland producer must bind the portal-selected window, its pixel stream, and diff --git a/openadapt_capture/control.py b/openadapt_capture/control.py index 68547be..4ca75e5 100644 --- a/openadapt_capture/control.py +++ b/openadapt_capture/control.py @@ -70,6 +70,7 @@ class RecorderStatus: integrity_verified: bool event_counts: dict[str, int] error_code: str | None = None + failure_stage: str | None = None @classmethod def from_payload(cls, payload: dict[str, Any]) -> "RecorderStatus": @@ -95,6 +96,11 @@ def from_payload(cls, payload: dict[str, Any]) -> "RecorderStatus": error_code=( str(payload["error_code"]) if payload.get("error_code") is not None else None ), + failure_stage=( + str(payload["failure_stage"]) + if payload.get("failure_stage") is not None + else None + ), ) except (KeyError, TypeError, ValueError) as exc: raise CaptureControlAuthenticationError( @@ -1193,6 +1199,7 @@ def _response( "integrity_verified", "event_counts", "error_code", + "failure_stage", ): if key in status: response[key] = status[key] diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 996484d..35c9f68 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -86,6 +86,8 @@ class WindowCaptureStateV2(BaseModel): pid: int = Field(gt=0) process_start_time: float = Field(gt=0) coordinate_source: str = Field(min_length=1) + capture_source: str = Field(default="platform-window-image", min_length=1) + visibility_independent: bool = False geometry_generation: int = Field(ge=1) geometry_epoch_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") display_topology_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") diff --git a/openadapt_capture/input_observer/darwin.py b/openadapt_capture/input_observer/darwin.py index ee95548..4f2e3d9 100644 --- a/openadapt_capture/input_observer/darwin.py +++ b/openadapt_capture/input_observer/darwin.py @@ -91,6 +91,8 @@ "ctrl_r": "ctrl", } +_MODIFIER_KEYCODES = frozenset({54, 55, 56, 57, 58, 59, 60, 61, 62, 63}) + class DarwinInputObserver(ThreadedInputObserver): """Observe macOS keyboard and mouse input without modifying the event stream.""" @@ -387,6 +389,15 @@ def _handle_event( timestamp: float | None = None, ) -> None: quartz = self._quartz + if event_type == quartz.kCGEventFlagsChanged and self.observe_keyboard: + keycode = self._keycode(event) + if keycode not in _MODIFIER_KEYCODES: + # Quartz can emit flagsChanged with keycode 0 when it cannot + # attribute the flag transition to one physical modifier. + # The event must invalidate an in-flight frame, but it cannot + # become an honest keyboard action. + self._mark_native_activity() + return injected = self._is_injected(event) # Production callbacks always pass the receipt time. Keeping ``None`` # for direct normalization calls makes the pure helper independently diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 6b5f0a8..e244895 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -75,7 +75,9 @@ observe_structural_action, ) from openadapt_capture.window_capture import ( + WindowCaptureAmbiguousError, WindowCaptureError, + WindowCapturePermissionError, WindowCaptureScope, build_window_scope, ) @@ -531,6 +533,28 @@ def __bool__(self): STARTUP_WAIT_POLL_SECONDS = 0.1 STARTUP_READY_TIMEOUT_SECONDS = 30.0 + +def _stable_recorder_error_code(exc: BaseException, stage: str) -> str: + """Return a privacy-safe failure code for the owner and control clients.""" + if isinstance(exc, WindowCapturePermissionError): + return "screen_capture_permission_denied" + if isinstance(exc, WindowCaptureAmbiguousError): + return "window_target_ambiguous" + if isinstance(exc, WindowCaptureError): + return "window_capture_failed" + if isinstance(exc, InputObserverError): + return "input_observer_failed" + if isinstance(exc, (video.FFmpegEncodingError, video.FFmpegUnavailableError)): + return "video_encoder_failed" + return { + "database_finalization": "database_finalization_failed", + "capture_verification": "capture_verification_failed", + "terminal_metadata": "terminal_metadata_failed", + "capture_seal": "capture_seal_failed", + "terminal_publish": "terminal_publish_failed", + "worker_health": "recording_worker_failed", + }.get(stage, "recording_failed") + # A writer announces readiness only after its first database write returns, so # the database's wait for the write lock is spent inside the deadline above. # Check the two against each other here rather than trusting a comment beside @@ -3408,6 +3432,7 @@ def __init__( self._control_complete = False self._control_integrity_verified = False self._control_error_code: str | None = None + self._control_failure_stage: str | None = None self._control_started_at = time.time() self._control_finalized_at: float | None = None @@ -3421,10 +3446,13 @@ def _control_payload(self) -> dict[str, Any]: "process_started_at": self._process_started_at, "capture_dir": self.capture_dir, "phase": self._control_phase, - "ready": self._ready_event.is_set(), + "ready": ( + self._control_phase == "recording" and self._ready_event.is_set() + ), "complete": self._control_complete, "integrity_verified": self._control_integrity_verified, "error_code": self._control_error_code, + "failure_stage": self._control_failure_stage, "started_at": self._control_started_at, "finalized_at": self._control_finalized_at, "event_counts": { @@ -3463,6 +3491,7 @@ def _stage_completed_control_state(self) -> float: "complete": True, "integrity_verified": True, "error_code": None, + "failure_stage": None, "finalized_at": finalized_at, } ) @@ -3483,6 +3512,7 @@ def _publish_completed_control_state(self, finalized_at: float) -> None: self._control_complete = True self._control_integrity_verified = True self._control_error_code = None + self._control_failure_stage = None self._control_finalized_at = finalized_at def _transition_control( @@ -3492,6 +3522,7 @@ def _transition_control( complete: bool = False, integrity_verified: bool = False, error_code: str | None = None, + failure_stage: str | None = None, finalized: bool = False, ) -> None: with self._control_state_lock: @@ -3502,6 +3533,7 @@ def _transition_control( self._control_complete, self._control_integrity_verified, self._control_error_code, + self._control_failure_stage, self._control_finalized_at, ) if ( @@ -3514,6 +3546,7 @@ def _transition_control( self._control_complete = complete self._control_integrity_verified = integrity_verified self._control_error_code = error_code + self._control_failure_stage = failure_stage if finalized: self._control_finalized_at = time.time() try: @@ -3524,6 +3557,7 @@ def _transition_control( self._control_complete, self._control_integrity_verified, self._control_error_code, + self._control_failure_stage, self._control_finalized_at, ) = previous raise @@ -3738,6 +3772,7 @@ def _run_record(self) -> None: """Thread target: apply config overrides, then call record().""" from openadapt_capture.config import config_override + failure_stage = "recording_startup" try: with config_override(self._recording_config): last_source_ordinal = record( @@ -3757,21 +3792,28 @@ def _run_record(self) -> None: ) if last_source_ordinal is not None: self._last_source_ordinal = last_source_ordinal + failure_stage = "worker_health" self.check_health() if self._ready_event.is_set(): # Every writer has exited by here, so fold the write log back # into the database before anything reads or inventories it. + failure_stage = "database_finalization" finalize_capture_database( os.path.join(self.capture_dir, "recording.db") ) + failure_stage = "capture_verification" self._verify_completed_capture() + failure_stage = "terminal_metadata" finalized_at = self._stage_completed_control_state() + failure_stage = "capture_seal" self._seal_completed_capture() + failure_stage = "terminal_publish" self._publish_completed_control_state(finalized_at) else: self._transition_control( "failed", error_code="startup_incomplete", + failure_stage="recording_startup", finalized=True, ) except BaseException as exc: @@ -3782,7 +3824,8 @@ def _run_record(self) -> None: try: self._transition_control( "failed", - error_code="recording_or_finalization_failed", + error_code=_stable_recorder_error_code(exc, failure_stage), + failure_stage=failure_stage, finalized=True, ) except BaseException as state_exc: diff --git a/openadapt_capture/video.py b/openadapt_capture/video.py index 21bc9ee..d959235 100644 --- a/openadapt_capture/video.py +++ b/openadapt_capture/video.py @@ -838,6 +838,13 @@ def _encode_command(self) -> list[str]: def _start(self) -> subprocess.Popen[bytes]: stderr_file = tempfile.TemporaryFile(mode="w+b") + process_group_kwargs: dict[str, object] + if os.name == "nt": + process_group_kwargs = { + "creationflags": subprocess.CREATE_NEW_PROCESS_GROUP, + } + else: + process_group_kwargs = {"start_new_session": True} try: process = subprocess.Popen( self._encode_command(), @@ -846,6 +853,7 @@ def _start(self) -> subprocess.Popen[bytes]: stderr=stderr_file, bufsize=0, shell=False, + **process_group_kwargs, ) except OSError as exc: stderr_file.close() diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 9996996..ce26993 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -3,8 +3,8 @@ Why this exists (the Citrix / remote-display wedge): openadapt-flow's ``RemoteDisplayBackend`` (``record --backend rdp``, ``rdp_window`` mode) *replays* against the pixels of a single client window (Parallels, Citrix -Workspace, Microsoft Remote Desktop) captured per-window — on macOS via -``CGWindowListCreateImage`` by window id. A demonstration recorded FULL-SCREEN +Workspace, Microsoft Remote Desktop) captured by exact window id. A +demonstration recorded FULL-SCREEN is in a different coordinate space than that replay surface, so converters had to work around the mismatch (record inside the session, or full-screen the client). Window-scoped recording removes the mismatch at the source: frames @@ -16,9 +16,9 @@ - ``bounds`` is ``(x, y, w, h)`` in **screen points**, top-left origin — the space native platform observers use for global mouse coordinates. -- A captured frame contains the window's own **pixels** (macOS: - ``CGWindowListCreateImage`` with ``kCGWindowImageBoundsIgnoreFraming`` — the - identical call flow's replay capture path uses). +- A captured frame contains the window's own **pixels**. Current macOS uses a + ScreenCaptureKit desktop-independent window filter. Exact-window Quartz and + system-utility providers remain compatibility paths. - ``scale`` = captured pixel width / bounds width (e.g. 2.0 on Retina). - Replay maps a captured pixel to a screen point as ``screen = bounds_origin + pixel / scale`` (flow's ``_to_screen``). @@ -36,7 +36,9 @@ import hashlib import json import math +import subprocess import sys +import tempfile import threading from contextlib import contextmanager from dataclasses import dataclass @@ -95,6 +97,18 @@ class WindowCaptureError(RuntimeError): """ +class WindowCapturePermissionError(WindowCaptureError): + """The operating system denied exact-window capture.""" + + +class WindowCaptureAmbiguousError(WindowCaptureError): + """The configured selectors match more than one capturable window.""" + + +class WindowCaptureUnavailableError(WindowCaptureError): + """The exact-window capture provider is not available.""" + + @dataclass(frozen=True) class WindowTarget: """How to find the window to record: case-insensitive substrings. @@ -153,6 +167,8 @@ class TargetWindow: on_screen: bool = True process_start_time: float | None = None coordinate_source: str = "platform-screen" + capture_source: str = "platform-window-image" + visibility_independent: bool = False @property def identity(self) -> tuple[int, int, float | None, str]: @@ -222,6 +238,7 @@ def __init__( self._source_viewport: tuple[int, int] | None = None self._content_rect: tuple[int, int, int, int] | None = None self._fit_scale: float | None = None + self._capture_source: str | None = None self._geometry_generation = 0 self._geometry_signature: tuple | None = None self._published_generation = 0 @@ -271,14 +288,14 @@ def resolve(self) -> TargetWindow: """Resolve the target window without changing captured-frame geometry. Raises: - WindowCaptureError: If no matching window is on screen. + WindowCaptureError: If no matching window can be captured safely. """ win = self._resolver(self.target) if win is None: raise WindowCaptureError( f"no window matching owner {self.target.owner!r} " f"title {self.target.title!r}; is the target application " - "running with a visible window?" + "running with a capturable window?" ) if self.target.owner and self.target.owner.casefold() not in win.owner.casefold(): raise WindowCaptureError( @@ -288,8 +305,11 @@ def resolve(self) -> TargetWindow: raise WindowCaptureError( "the window resolver returned a title outside the configured selector" ) - if not win.on_screen: - raise WindowCaptureError("the resolved target window is not on screen") + if not win.on_screen and not win.visibility_independent: + raise WindowCaptureError( + "the resolved target window is not visible and the active " + "capture provider requires visibility" + ) if win.pid <= 0: raise WindowCaptureError("the resolved target has no owning process identity") if ( @@ -385,6 +405,9 @@ def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: win = post if source_image.width <= 0 or source_image.height <= 0: raise WindowCaptureError("window capture returned an empty frame") + capture_source = str( + source_image.info.get("openadapt_capture_source", win.capture_source) + ) source_viewport = (source_image.width, source_image.height) output_viewport = output_viewport or source_viewport output_width, output_height = output_viewport @@ -436,6 +459,7 @@ def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: self._source_viewport = source_viewport self._content_rect = (offset_x, offset_y, fitted_width, fitted_height) self._fit_scale = fit_scale + self._capture_source = capture_source if self._bound_identity is None: self._bound_identity = win.identity if geometry_signature != self._geometry_signature: @@ -510,8 +534,11 @@ def _geometry_for_action( ) live = self.resolve() self._assert_bound_identity(live) - if not live.on_screen: - raise WindowCaptureError("the target window is not on screen at action time") + if not live.on_screen and not live.visibility_independent: + raise WindowCaptureError( + "the target window is not visible at action time and the active " + "capture provider requires visibility" + ) if live.bounds != window.bounds: raise WindowCaptureError( "the target moved or resized after the last published frame; " @@ -610,6 +637,7 @@ def window_event_data(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + capture_source = self._capture_source generation = self._geometry_generation topology = self._display_topology if window is None: @@ -623,6 +651,8 @@ def window_event_data(self) -> dict: "pid": window.pid, "process_start_time": window.process_start_time, "coordinate_source": window.coordinate_source, + "capture_source": capture_source or window.capture_source, + "visibility_independent": window.visibility_independent, "geometry_generation": generation, "display_topology_sha256": ( topology.get("topology_sha256") if topology else None @@ -664,6 +694,7 @@ def snapshot(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + capture_source = self._capture_source generation = self._geometry_generation topology = self._display_topology data: dict = { @@ -681,6 +712,8 @@ def snapshot(self) -> dict: "pid": window.pid, "process_start_time": window.process_start_time, "coordinate_source": window.coordinate_source, + "capture_source": capture_source or window.capture_source, + "visibility_independent": window.visibility_independent, "geometry_generation": generation, "initial_bounds": list(window.bounds), "scale": scale, @@ -802,19 +835,222 @@ def _process_start_time(pid: int) -> float: ) from exc +_MACOS_CAPTURE_TIMEOUT_SECONDS = 15.0 +_MACOS_SC_WINDOW_CACHE: dict[int, object] = {} +_MACOS_SC_WINDOW_CACHE_LOCK = threading.Lock() + + +def _screen_capture_kit_available() -> bool: + """Return whether single-frame desktop-independent capture is available.""" + try: + import ScreenCaptureKit + except ImportError: + return False + return hasattr( + ScreenCaptureKit.SCScreenshotManager, + "captureImageWithFilter_configuration_completionHandler_", + ) + + +def _macos_completion( + start: Callable[[Callable[..., None]], None], + *, + operation: str, +) -> object: + """Wait for one ScreenCaptureKit completion without requiring an event loop.""" + completed = threading.Event() + result: dict[str, object | None] = {"value": None, "error": None} + + def completion(value: object | None, error: object | None) -> None: + result["value"] = value + result["error"] = error + completed.set() + + try: + start(completion) + except Exception as exc: + raise WindowCaptureUnavailableError( + f"ScreenCaptureKit could not start {operation}" + ) from exc + if not completed.wait(_MACOS_CAPTURE_TIMEOUT_SECONDS): + raise WindowCaptureUnavailableError( + f"ScreenCaptureKit timed out during {operation}" + ) + if result["error"] is not None: + error = result["error"] + code_getter = getattr(error, "code", None) + code = int(code_getter()) if callable(code_getter) else None + error_type = ( + WindowCapturePermissionError + if code in {-3801, -3803} + else WindowCaptureError + ) + raise error_type( + f"ScreenCaptureKit failed during {operation}" + + (f" (error {code})" if code is not None else "") + ) + if result["value"] is None: + raise WindowCaptureError( + f"ScreenCaptureKit returned no result during {operation}" + ) + return result["value"] + + +def _screen_capture_kit_window(window_id: int) -> object: + """Return the exact shareable window, including windows on other Spaces.""" + with _MACOS_SC_WINDOW_CACHE_LOCK: + cached = _MACOS_SC_WINDOW_CACHE.get(window_id) + if cached is not None: + return cached + + try: + import ScreenCaptureKit + except ImportError as exc: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit requires pyobjc-framework-ScreenCaptureKit" + ) from exc + + get_content = getattr( + ScreenCaptureKit.SCShareableContent, + "getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHandler_", + ) + content = _macos_completion( + lambda callback: get_content( + True, + False, + callback, + ), + operation="window enumeration", + ) + windows = list(content.windows() or []) + match = next( + (candidate for candidate in windows if int(candidate.windowID()) == window_id), + None, + ) + if match is None: + raise WindowCaptureError( + f"ScreenCaptureKit cannot access exact window {window_id}" + ) + with _MACOS_SC_WINDOW_CACHE_LOCK: + _MACOS_SC_WINDOW_CACHE[window_id] = match + return match + + +def _pil_image_from_cgimage(img_ref: object, *, source: str) -> "Image.Image": + """Convert one ScreenCaptureKit or Quartz CGImage to stable RGB pixels.""" + import Quartz + from PIL import Image + + width = int(Quartz.CGImageGetWidth(img_ref)) + height = int(Quartz.CGImageGetHeight(img_ref)) + if width <= 0 or height <= 0: + raise WindowCaptureError("captured window image has zero size") + bytes_per_row = int(Quartz.CGImageGetBytesPerRow(img_ref)) + provider = Quartz.CGImageGetDataProvider(img_ref) + data = Quartz.CGDataProviderCopyData(provider) + image = Image.frombuffer( + "RGBA", + (width, height), + bytes(data), + "raw", + "BGRA", + bytes_per_row, + 1, + ).convert("RGB") + image.info["openadapt_capture_source"] = source + return image + + +def _capture_window_macos_screencapturekit(window: TargetWindow) -> "Image.Image": + """Capture one exact window without requiring it to be frontmost or visible.""" + try: + import ScreenCaptureKit + except ImportError as exc: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit requires pyobjc-framework-ScreenCaptureKit" + ) from exc + + sc_window = _screen_capture_kit_window(window.window_id) + content_filter = ScreenCaptureKit.SCContentFilter.alloc().initWithDesktopIndependentWindow_( + sc_window + ) + point_scale = float(content_filter.pointPixelScale()) + if not math.isfinite(point_scale) or point_scale <= 0: + raise WindowCaptureError("ScreenCaptureKit returned an invalid point-to-pixel scale") + configuration = ScreenCaptureKit.SCStreamConfiguration.alloc().init() + configuration.setWidth_(max(1, round(window.bounds[2] * point_scale))) + configuration.setHeight_(max(1, round(window.bounds[3] * point_scale))) + configuration.setShowsCursor_(False) + if hasattr(configuration, "setIgnoreShadowsSingleWindow_"): + configuration.setIgnoreShadowsSingleWindow_(True) + capture_image = getattr( + ScreenCaptureKit.SCScreenshotManager, + "captureImageWithFilter_configuration_completionHandler_", + ) + img_ref = _macos_completion( + lambda callback: capture_image( + content_filter, + configuration, + callback, + ), + operation=f"exact-window capture for window {window.window_id}", + ) + return _pil_image_from_cgimage(img_ref, source="macos-screencapturekit") + + +def _capture_window_macos_utility(window: TargetWindow) -> "Image.Image": + """Use the signed system utility as an exact-window compatibility path.""" + from PIL import Image + + with tempfile.TemporaryDirectory(prefix="openadapt-window-") as temp_dir: + capture_path = f"{temp_dir}/window.png" + try: + result = subprocess.run( + [ + "/usr/sbin/screencapture", + "-x", + "-o", + "-l", + str(window.window_id), + capture_path, + ], + check=False, + capture_output=True, + text=True, + timeout=_MACOS_CAPTURE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise WindowCaptureUnavailableError( + "the macOS exact-window compatibility capture could not run" + ) from exc + if result.returncode != 0: + raise WindowCapturePermissionError( + "the macOS exact-window compatibility capture was denied" + ) + try: + with Image.open(capture_path) as image: + captured = image.convert("RGB").copy() + except OSError as exc: + raise WindowCaptureError( + "the macOS exact-window compatibility capture produced no readable image" + ) from exc + captured.info["openadapt_capture_source"] = "macos-screencapture-utility" + return captured + + def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: """macOS: CGWindowList by owner/title substring. - Selects the largest visible layer-0 window that matches the configured - owner/title substrings. + ScreenCaptureKit can capture an occluded window or a window on another + Space. Older macOS versions retain the visible-window Quartz behavior. """ import Quartz + visibility_independent = _screen_capture_kit_available() owner_l = target.owner.lower() if target.owner else None title_l = target.title.lower() if target.title else None wins = Quartz.CGWindowListCopyWindowInfo(Quartz.kCGWindowListOptionAll, Quartz.kCGNullWindowID) - best: TargetWindow | None = None - best_area = -1.0 + matches: list[TargetWindow] = [] for w in wins or []: owner = str(w.get("kCGWindowOwnerName", "") or "") name = str(w.get("kCGWindowName", "") or "") @@ -824,7 +1060,8 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: continue if int(w.get("kCGWindowLayer", 0) or 0) != 0: continue # skip menubar/overlay layers; the app window is layer 0 - if not bool(w.get("kCGWindowIsOnscreen", False)): + on_screen = bool(w.get("kCGWindowIsOnscreen", False)) + if not on_screen and not visibility_independent: continue b = w.get("kCGWindowBounds", {}) or {} bounds = ( @@ -833,58 +1070,100 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: float(b.get("Width", 0.0)), float(b.get("Height", 0.0)), ) - area = bounds[2] * bounds[3] - if area > best_area: - best_area = area - pid = int(w.get("kCGWindowOwnerPID", 0) or 0) - best = TargetWindow( + if bounds[2] <= 0 or bounds[3] <= 0: + continue + pid = int(w.get("kCGWindowOwnerPID", 0) or 0) + matches.append( + TargetWindow( window_id=int(w.get("kCGWindowNumber", 0) or 0), owner=owner, title=name, pid=pid, bounds=bounds, - on_screen=bool(w.get("kCGWindowIsOnscreen", False)), + on_screen=on_screen, process_start_time=_process_start_time(pid), coordinate_source="quartz-screen-points", + capture_source=( + "macos-exact-window-provider-chain" + if visibility_independent + else "macos-quartz-window-image" + ), + visibility_independent=visibility_independent, ) - return best + ) + if not matches: + return None + + exact = [ + match + for match in matches + if (target.owner is None or match.owner.casefold() == target.owner.casefold()) + and (target.title is None or match.title.casefold() == target.title.casefold()) + ] + candidates = exact or matches + if len(candidates) != 1: + raise WindowCaptureAmbiguousError( + f"window selectors matched {len(candidates)} capturable windows; " + "provide the complete window title" + ) + return candidates[0] def _capture_window_macos(window: TargetWindow) -> "Image.Image": - """macOS: per-window capture via ``CGWindowListCreateImage``. + """Capture an exact macOS window through an explicit provider chain. - The identical call (``CGRectNull`` + ``kCGWindowListOptionIncludingWindow`` - + ``kCGWindowImageBoundsIgnoreFraming``) and BGRA->RGB conversion as flow's - ``MacWindowClient.capture`` — the replay surface — so recorded frames and - replay frames share coordinate semantics byte for byte. + ScreenCaptureKit is primary. Quartz remains for older systems. The signed + system utility is the final exact-window compatibility path. No provider + can substitute a full-screen image or another window. """ import Quartz - from PIL import Image - img_ref = Quartz.CGWindowListCreateImage( - Quartz.CGRectNull, - Quartz.kCGWindowListOptionIncludingWindow, - window.window_id, - Quartz.kCGWindowImageBoundsIgnoreFraming, + failures: list[BaseException] = [] + if _screen_capture_kit_available(): + try: + return _capture_window_macos_screencapturekit(window) + except WindowCaptureError as exc: + failures.append(exc) + logger.warning( + "ScreenCaptureKit exact-window capture failed; trying the " + "legacy exact-window providers" + ) + + img_ref = None + if window.on_screen: + img_ref = Quartz.CGWindowListCreateImage( + Quartz.CGRectNull, + Quartz.kCGWindowListOptionIncludingWindow, + window.window_id, + Quartz.kCGWindowImageBoundsIgnoreFraming, + ) + if img_ref is not None: + return _pil_image_from_cgimage(img_ref, source="macos-quartz-window-image") + failures.append( + WindowCapturePermissionError( + ( + "Quartz returned no image" + if window.on_screen + else "Quartz was skipped because the target is not on screen" + ) + + f" for exact window {window.window_id}" + ) ) - if img_ref is None: - raise WindowCaptureError( - f"CGWindowListCreateImage returned None for window " - f"{window.window_id} ({window.owner!r}); if this recurs, check " - "Screen Recording permission for the recording process" + try: + return _capture_window_macos_utility(window) + except WindowCaptureError as exc: + failures.append(exc) + failure = WindowCaptureError( + f"all exact-window capture providers failed for window {window.window_id}" ) - w = int(Quartz.CGImageGetWidth(img_ref)) - h = int(Quartz.CGImageGetHeight(img_ref)) - if w <= 0 or h <= 0: - raise WindowCaptureError("captured window image has zero size") - bpr = int(Quartz.CGImageGetBytesPerRow(img_ref)) - provider = Quartz.CGImageGetDataProvider(img_ref) - data = Quartz.CGDataProviderCopyData(provider) - buf = bytes(data) - # CGImage from the window server is BGRA, premultiplied; read with the row - # stride and drop alpha for a stable RGB frame. - img = Image.frombuffer("RGBA", (w, h), buf, "raw", "BGRA", bpr, 1) - return img.convert("RGB") + for provider_failure in failures: + try: + failure.add_note( + f"{type(provider_failure).__name__}: {provider_failure}" + ) + except AttributeError: # Python 3.10 + pass + raise failure from exc def _resolve_window_windows(target: WindowTarget) -> TargetWindow | None: diff --git a/pyproject.toml b/pyproject.toml index 1f2f1bd..2da3ee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,8 @@ dependencies = [ "matplotlib>=3.10.8", # Native macOS window/accessibility observation via permissive PyObjC. "pyobjc-framework-ApplicationServices>=12.2.1; sys_platform == 'darwin'", + # Desktop-independent, exact-window pixels on macOS 14 and later. + "pyobjc-framework-ScreenCaptureKit>=12.2.1; sys_platform == 'darwin'", # Native Windows structural observation. Platform-gated so non-Windows # installs and imports do not load the Windows UIA stack. "pywinauto>=0.6.9; sys_platform == 'win32'", diff --git a/tests/test_control.py b/tests/test_control.py index 00d926d..7b24bc2 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -703,9 +703,11 @@ def fail_after_ready(*, status_pipe, **_kwargs) -> None: (capture_dir / control.TERMINAL_STATE_FILENAME).read_text(encoding="utf-8") ) assert terminal["phase"] == "failed" + assert terminal["ready"] is False assert terminal["complete"] is False assert terminal["integrity_verified"] is False - assert terminal["error_code"] == "recording_or_finalization_failed" + assert terminal["error_code"] == "recording_failed" + assert terminal["failure_stage"] == "recording_startup" def test_complete_state_write_failure_cannot_return_success( diff --git a/tests/test_input_observer_darwin.py b/tests/test_input_observer_darwin.py index 3c3e630..81699f6 100644 --- a/tests/test_input_observer_darwin.py +++ b/tests/test_input_observer_darwin.py @@ -716,6 +716,21 @@ def test_key_press_release_and_modifier_canonicalization() -> None: ] +def test_unattributable_flags_changed_is_ignored_without_failing_observer() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + + observer.start() + event = FakeEvent(fields={quartz.kCGKeyboardEventKeycode: 0}) + assert observer._event_callback(None, quartz.kCGEventFlagsChanged, event, None) is event + observer.check_health() + observer.stop() + + assert events == [] + assert observer._input_receipt_count == 1 + + def test_first_observed_modifier_release_does_not_toggle_to_press() -> None: quartz = FakeQuartz() events = [] diff --git a/tests/test_video.py b/tests/test_video.py index f509a4b..831af00 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -5,6 +5,7 @@ import io import json import multiprocessing +import os import shutil import subprocess import time @@ -389,6 +390,10 @@ def test_direct_stream_preserves_pts_timing_without_png_staging(tmp_path, monkey def popen(command, **kwargs): assert kwargs["shell"] is False + if os.name == "nt": + assert kwargs["creationflags"] == subprocess.CREATE_NEW_PROCESS_GROUP + else: + assert kwargs["start_new_session"] is True process = _FakeProcess(list(command)) kwargs["stderr"].flush() processes.append(process) diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index c7aabf7..1e8831b 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -43,7 +43,9 @@ ) from openadapt_capture.window_capture import ( TargetWindow, + WindowCaptureAmbiguousError, WindowCaptureError, + WindowCapturePermissionError, WindowCaptureScope, WindowTarget, build_window_scope, @@ -911,6 +913,7 @@ def test_window_capture_state_rejects_scales_not_derived_from_content(scope): def test_macos_resolver_ignores_a_larger_hidden_matching_window(monkeypatch): + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) hidden = { "kCGWindowOwnerName": "FakeApp", "kCGWindowName": "Document", @@ -944,6 +947,128 @@ def test_macos_resolver_ignores_a_larger_hidden_matching_window(monkeypatch): assert resolved.on_screen is True +def test_macos_resolver_accepts_offspace_window_with_screencapturekit(monkeypatch): + hidden = { + "kCGWindowOwnerName": "FakeApp", + "kCGWindowName": "Document", + "kCGWindowLayer": 0, + "kCGWindowIsOnscreen": False, + "kCGWindowBounds": {"X": 0, "Y": 0, "Width": 1512, "Height": 944}, + "kCGWindowOwnerPID": 100, + "kCGWindowNumber": 19373, + } + quartz = SimpleNamespace( + kCGWindowListOptionAll=1, + kCGNullWindowID=0, + CGWindowListCopyWindowInfo=lambda *_args: [hidden], + ) + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_process_start_time", lambda _pid: 123.0) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + + resolved = window_capture_module._resolve_window_macos( + WindowTarget(owner="FakeApp", title="Document") + ) + + assert resolved is not None + assert resolved.window_id == 19373 + assert resolved.on_screen is False + assert resolved.visibility_independent is True + assert resolved.capture_source == "macos-exact-window-provider-chain" + + +def test_macos_resolver_refuses_ambiguous_owner_only_match(monkeypatch): + windows = [ + { + "kCGWindowOwnerName": "Google Chrome", + "kCGWindowName": title, + "kCGWindowLayer": 0, + "kCGWindowIsOnscreen": True, + "kCGWindowBounds": {"X": 0, "Y": 0, "Width": 1512, "Height": 944}, + "kCGWindowOwnerPID": 100, + "kCGWindowNumber": window_id, + } + for window_id, title in ((95, "Profile picker"), (19373, "Amex")) + ] + quartz = SimpleNamespace( + kCGWindowListOptionAll=1, + kCGNullWindowID=0, + CGWindowListCopyWindowInfo=lambda *_args: windows, + ) + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_process_start_time", lambda _pid: 123.0) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + + with pytest.raises(WindowCaptureAmbiguousError, match="complete window title"): + window_capture_module._resolve_window_macos(WindowTarget(owner="Google Chrome")) + + +def test_macos_capture_uses_exact_utility_after_sck_and_quartz_fail(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + capture_source="macos-exact-window-provider-chain", + visibility_independent=True, + ) + quartz = SimpleNamespace() + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_screencapturekit", + lambda _window: (_ for _ in ()).throw(WindowCapturePermissionError("denied")), + ) + expected = Image.new("RGB", (3024, 1888), "white") + expected.info["openadapt_capture_source"] = "macos-screencapture-utility" + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda _window: expected, + ) + + captured = window_capture_module._capture_window_macos(window) + + assert captured.size == (3024, 1888) + assert captured.info["openadapt_capture_source"] == "macos-screencapture-utility" + + +def test_screencapturekit_enumerates_other_spaces_and_selects_exact_id(monkeypatch): + calls = [] + + class FakeWindow: + def __init__(self, window_id): + self._window_id = window_id + + def windowID(self): + return self._window_id + + expected = FakeWindow(19373) + content = SimpleNamespace(windows=lambda: [FakeWindow(95), expected]) + + class FakeShareableContent: + @staticmethod + def getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHandler_( + exclude_desktop, on_screen_only, callback + ): + calls.append((exclude_desktop, on_screen_only)) + callback(content, None) + + fake_sck = SimpleNamespace(SCShareableContent=FakeShareableContent) + monkeypatch.setitem(sys.modules, "ScreenCaptureKit", fake_sck) + window_capture_module._MACOS_SC_WINDOW_CACHE.clear() + + resolved = window_capture_module._screen_capture_kit_window(19373) + + assert resolved is expected + assert calls == [(True, False)] + + class TestTranslatePoint: """Coordinate translation: global screen points -> window pixels.""" diff --git a/uv.lock b/uv.lock index de54890..c7296d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1896,6 +1896,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pympler" }, { name = "pyobjc-framework-applicationservices", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-screencapturekit", marker = "sys_platform == 'darwin'" }, { name = "pywinauto", marker = "sys_platform == 'win32'" }, { name = "sounddevice" }, { name = "soundfile" }, @@ -1966,6 +1967,7 @@ requires-dist = [ { name = "pygobject", marker = "sys_platform == 'linux' and extra == 'linux'", specifier = ">=3.46,<3.50" }, { name = "pympler", specifier = ">=1.0.0" }, { name = "pyobjc-framework-applicationservices", marker = "sys_platform == 'darwin'", specifier = ">=12.2.1" }, + { name = "pyobjc-framework-screencapturekit", marker = "sys_platform == 'darwin'", specifier = ">=12.2.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, @@ -2483,6 +2485,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/3b/07ce3c0ab8d1e9e1bed74fea1bf1cce73527a365a7a23c755051d3be9865/pyobjc_framework_cocoa-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8fe5b2e79c9530f667b4e58a87a3a15ea62f86a5d19eec405517ecbd4f454868", size = 392693, upload-time = "2026-08-11T19:32:50.283Z" }, ] +[[package]] +name = "pyobjc-framework-coremedia" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/12/0b3896ea04f0fbbe3f2cb37802cadce334f9aef204d911a81934013eba8c/pyobjc_framework_coremedia-12.2.2.tar.gz", hash = "sha256:fe9f972438674893e941e9db5b62f8ba99a33f94a7f1c2ce49b14416f154cd10", size = 98580, upload-time = "2026-08-11T19:44:11.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/7a/de17f374501561a30f50a9c194bfaf39607c717cad062057ba16b209e79e/pyobjc_framework_coremedia-12.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41142c919c557165df4a53ffe218e600ad0ccb7691bc9aed58a7839a88ba7c8b", size = 29520, upload-time = "2026-08-11T19:33:48.193Z" }, + { url = "https://files.pythonhosted.org/packages/38/18/5931478200f803db87be5cd6cb9af492d9ef2f2ac07cbfe019ecfa97db5a/pyobjc_framework_coremedia-12.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f6f8150428e6ae40d6b97714f6933b8c4322f3fb94de9bfcb206d00cb021f464", size = 29518, upload-time = "2026-08-11T19:33:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/cd/26/de907d6ead970916b39787e52bed679945328161f2abac533515b2ac8934/pyobjc_framework_coremedia-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:84f5deee8d366bf73069a0525509637aa578ccd0e6688923b09999a8bc3077a8", size = 29421, upload-time = "2026-08-11T19:33:49.848Z" }, + { url = "https://files.pythonhosted.org/packages/36/65/6fe4c97d3442abda385f8798becd5bc82b73b4dfd979194e5850c5246dea/pyobjc_framework_coremedia-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c4430bda640c5427bf572f8b2a3281d8f4f16b962a499251cf553a0979d88a4e", size = 29431, upload-time = "2026-08-11T19:33:50.621Z" }, + { url = "https://files.pythonhosted.org/packages/5f/32/3c023f6b26fffb8c7d07cd6327e892bd546da7cfa8b3bbdaed113f5a59d3/pyobjc_framework_coremedia-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b80285a65465a5340f5806e9e293b46e22ac6f9c3132e79c8139a79b53b3f8c2", size = 29496, upload-time = "2026-08-11T19:33:51.396Z" }, + { url = "https://files.pythonhosted.org/packages/74/bb/0676dabebfbf9a89ea268ca26bd978a6c0e5dcf9b2baca86709bd8228708/pyobjc_framework_coremedia-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:edc9d03e5230123c92d4f0413c48449a7f08fa747b9f304b2cc396239acfefe3", size = 29465, upload-time = "2026-08-11T19:33:52.149Z" }, + { url = "https://files.pythonhosted.org/packages/17/33/1e2ab0438a546cc33ac18dcdb939c0a7cbbfcc070744f6451d08ccd80d6b/pyobjc_framework_coremedia-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:bb4c56edd411a5c0152a0a9eeb0138f9c173820041beff0cfaff291b9376bade", size = 29525, upload-time = "2026-08-11T19:33:52.943Z" }, + { url = "https://files.pythonhosted.org/packages/99/7f/1a5e92a337e924f4cf3781f61b30a0e9fba84eb6fbf2c377959bb842d1e8/pyobjc_framework_coremedia-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:49b53bbe1eea893f94bbf05f1f5220fcef3ccac7310a96d78434c0df6318dc79", size = 29503, upload-time = "2026-08-11T19:33:53.724Z" }, + { url = "https://files.pythonhosted.org/packages/db/f6/09e85dc8bffbafdcf74c53d8495db1411f9cc03b32a10593832f4fa81318/pyobjc_framework_coremedia-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf4cd65f87bd073e74b3ede2381dc9355529e6f1b12184da22090f016e68bf7", size = 29560, upload-time = "2026-08-11T19:33:54.712Z" }, +] + [[package]] name = "pyobjc-framework-coretext" version = "12.2.2" @@ -2526,6 +2549,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/eb/5fb627c2457046883c6fd12d25c44db40c12bcde4622dc0f30851108e106/pyobjc_framework_quartz-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8f58c589b5a76ba98f186b1f3b19fb1c8b730e82351f81fb62e6194f64a71622", size = 224767, upload-time = "2026-08-11T19:40:30.991Z" }, ] +[[package]] +name = "pyobjc-framework-screencapturekit" +version = "12.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-coremedia" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/b6/3a5b4a1d6410a06f5eafb17f17af55b1c7641455ca64ce55a9d05b72a096/pyobjc_framework_screencapturekit-12.2.2.tar.gz", hash = "sha256:6c16b1730186a012707e69532e6a03ae7282a6661f106db2663c0f0acc76420a", size = 38008, upload-time = "2026-08-11T19:45:19.875Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/0c/970adcb250389fc189cffa97b6ccc537aea227aa4d84d5a2064a5a043bc6/pyobjc_framework_screencapturekit-12.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ed3bdfbb6c6577bea4571c542143ad017d7fbc2592f94ced09e76003c78d2350", size = 11568, upload-time = "2026-08-11T19:41:02.239Z" }, + { url = "https://files.pythonhosted.org/packages/bc/3b/98dface3dc6de5ff56d048df8b65990b98ae6d174d6d1b025693dad99ceb/pyobjc_framework_screencapturekit-12.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0438a2b3fa99fbc7624e97850051d82dc7f5436748cfa3a8ee3179b21a1f545b", size = 11567, upload-time = "2026-08-11T19:41:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/05/17/1bf74d227db0abc7ee8cd1d5affcdf8f81cdf514d14f6ce938b94f3b3d0f/pyobjc_framework_screencapturekit-12.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5c81c68bc0399ad7ca6ccaa978574354a4da8fbb474527664fdbf08cc8f8abd5", size = 11598, upload-time = "2026-08-11T19:41:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/b9/b9/6faa6336f8fe8237585fb68485b2bd4e6a958ab024a3d2080c05699e230d/pyobjc_framework_screencapturekit-12.2.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:63fffcebe5998fb05c924aacfdd481ae49bb79fde0242e99780d33869f047c58", size = 11620, upload-time = "2026-08-11T19:41:04.564Z" }, + { url = "https://files.pythonhosted.org/packages/5a/fd/8524eed2eb852921e1e378e2963e5824bcf7b8022664deda869391fa197c/pyobjc_framework_screencapturekit-12.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5ae135d0ce93a70e32a5bb6a91b6fbaa3fbe159abd07e4c0eb8d8ce871ecffe", size = 11796, upload-time = "2026-08-11T19:41:05.286Z" }, + { url = "https://files.pythonhosted.org/packages/b5/76/b75fbd8a62c4335784c82ed0f2347805ab44d51c2a0bd9aa08cb8fc34882/pyobjc_framework_screencapturekit-12.2.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9c410ac6d7f41daaf772e7b3e9bd5057e094e77233aea7f7e18443d7e248e61", size = 11676, upload-time = "2026-08-11T19:41:06.099Z" }, + { url = "https://files.pythonhosted.org/packages/dd/e4/4427a313758e50ea06699a509cbde764a26c6cabfe6bc6bb763be69f0f1b/pyobjc_framework_screencapturekit-12.2.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:f6ac1e5d599303569c88a63c09e50ba289801bc40608afb3977b87c554566875", size = 11877, upload-time = "2026-08-11T19:41:06.842Z" }, + { url = "https://files.pythonhosted.org/packages/47/01/543afe3b5da651cea12b3fff9fc2638f4b551b7b6225c81222a8a4b0a20b/pyobjc_framework_screencapturekit-12.2.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:508147c240cddbde87149c2918c780fd3f896238ee290550565957a3c85dffc9", size = 11667, upload-time = "2026-08-11T19:41:07.574Z" }, + { url = "https://files.pythonhosted.org/packages/b5/ab/cfb7fad425532537b6620eba9e78c050cad4219fdd6a725bcf2e1c76e1aa/pyobjc_framework_screencapturekit-12.2.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:b052df28a13814b2b2eb25b7585224a4fcbd7ca1c5c711b485bb97be7789810f", size = 11870, upload-time = "2026-08-11T19:41:08.302Z" }, +] + [[package]] name = "pyopenssl" version = "26.4.0" From 17e1280e1ffff62faf04d843399ffdc8d0790cbe Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 30 Aug 2026 20:51:44 -0400 Subject: [PATCH 2/5] fix(macos): persist exact-window stream evidence --- README.md | 11 +- docs/WINDOW_CAPTURE.md | 16 +- openadapt_capture/capture.py | 33 +- openadapt_capture/events.py | 70 +++ openadapt_capture/recorder.py | 24 +- openadapt_capture/window_capture.py | 826 ++++++++++++++++++++++++---- tests/test_window_capture.py | 367 +++++++++++- 7 files changed, 1229 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 5ad10c3..142c8b9 100644 --- a/README.md +++ b/README.md @@ -213,10 +213,13 @@ producing media that looks complete but has an evidence gap. The full contract, including the multi-monitor rules and the coordinate-space flags converters must respect, is in [docs/WINDOW_CAPTURE.md](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/WINDOW_CAPTURE.md). -On current macOS, exact-window capture uses ScreenCaptureKit and does not -require the window to be frontmost. It can capture an occluded window or a -window on another Space. The recorder still needs a logged-in desktop session, -and a minimized window must return a valid exact frame. +On current macOS, exact-window capture keeps one ScreenCaptureKit stream bound +to a desktop-independent window filter. The window can be occluded or on +another Space. It does not need to be frontmost or reported as on screen. The +recorder accepts complete frames and proven idle frames. A failed provider is +disabled for the rest of that recording, so each frame does not repeat a slow +failure. The recorder still needs a logged-in desktop session. A minimized +window must return a valid exact frame. Linux window mode needs X11 with EWMH and XComposite. It refuses to start under native Wayland or XWayland-only. diff --git a/docs/WINDOW_CAPTURE.md b/docs/WINDOW_CAPTURE.md index 0cc8546..6c740cf 100644 --- a/docs/WINDOW_CAPTURE.md +++ b/docs/WINDOW_CAPTURE.md @@ -43,11 +43,17 @@ bind the recording to a different window. In this mode: -- **Frames are the target window's pixels.** On current macOS, ScreenCaptureKit - captures a desktop-independent exact-window filter. The legacy Quartz image - API and `/usr/sbin/screencapture -o -l` remain exact-window compatibility - paths. This supports an occluded window and a window on another Space. A - minimized window must still return a valid exact frame or the session fails. +- **Frames are the target window's pixels.** On current macOS, one persistent + ScreenCaptureKit stream stays bound to a desktop-independent exact-window + filter. It accepts complete frames and reuses the last complete pixels only + when ScreenCaptureKit reports that the window is idle. Each window event + retains the stream generation, sequence, frame status, display time, pixel + time, and a capture-evidence digest. The legacy Quartz + image API and `/usr/sbin/screencapture -o -l` remain exact-window + compatibility paths. A failed provider stays disabled for that recording. + This supports an occluded window and a window on another Space. It also + supports a window that macOS does not report as on screen. A minimized window + must still return a valid exact frame or the session fails. Linux X11 reads an XComposite named-window pixmap. It doesn't use a root screenshot, so another window cannot replace the target pixels. Windows grabs the window's screen region, so keep the window unoccluded. diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 3a71d5e..e6a4e80 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -1269,6 +1269,8 @@ def window_capture_events_v2(self) -> list[CapturedWindowEvent]: result = self.window_events() if not result: raise InvalidCaptureEvent("v2 window-scoped capture has no window events") + previous_stream_generation: int | None = None + previous_stream_sequence: int | None = None for event in result: state: WindowCaptureStateV2 | None = event.window_capture_v2 if state is None: @@ -1283,8 +1285,35 @@ def window_capture_events_v2(self) -> list[CapturedWindowEvent]: or event.height != int(height) ): raise InvalidCaptureEvent("stored WindowEvent columns differ from their v2 bounds") - if not state.on_screen: - raise InvalidCaptureEvent("v2 window event retained an off-screen target") + if not state.on_screen and not state.visibility_independent: + raise InvalidCaptureEvent( + "v2 window event retained an off-screen target without a " + "visibility-independent provider" + ) + if state.capture_source == "macos-screencapturekit-stream": + generation = state.stream_generation + sequence = state.stream_sequence + if generation is None or sequence is None: + raise InvalidCaptureEvent( + "ScreenCaptureKit frame evidence is incomplete" + ) + if ( + previous_stream_generation is not None + and generation < previous_stream_generation + ): + raise InvalidCaptureEvent( + "ScreenCaptureKit stream generation moved backward" + ) + if ( + previous_stream_generation == generation + and previous_stream_sequence is not None + and sequence <= previous_stream_sequence + ): + raise InvalidCaptureEvent( + "ScreenCaptureKit stream sequence is not increasing" + ) + previous_stream_generation = generation + previous_stream_sequence = sequence return result def actions(self, include_moves: bool = False) -> Iterator[Action]: diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 35c9f68..7fbf64f 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -16,6 +16,7 @@ from openadapt_capture.structural import StructuralObservation from openadapt_capture.window_capture import ( WINDOW_CAPTURE_SCHEMA_VERSION, + window_capture_evidence_sha256, window_geometry_epoch_sha256, ) @@ -88,6 +89,15 @@ class WindowCaptureStateV2(BaseModel): coordinate_source: str = Field(min_length=1) capture_source: str = Field(default="platform-window-image", min_length=1) visibility_independent: bool = False + frame_status: Literal["complete", "idle"] | None = None + frame_display_time: int | None = Field(default=None, ge=0) + pixel_display_time: int | None = Field(default=None, ge=0) + stream_generation: int | None = Field(default=None, ge=1) + stream_sequence: int | None = Field(default=None, ge=1) + capture_evidence_sha256: str | None = Field( + default=None, + pattern=r"^[0-9a-f]{64}$", + ) geometry_generation: int = Field(ge=1) geometry_epoch_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") display_topology_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") @@ -117,6 +127,55 @@ def _closed_geometry(self) -> "WindowCaptureStateV2": raise ValueError("window capture geometry must be finite") if self.bounds[2] <= 0 or self.bounds[3] <= 0: raise ValueError("window capture bounds must have positive dimensions") + if not self.on_screen: + if not self.visibility_independent: + raise ValueError( + "an off-screen window requires a visibility-independent provider" + ) + if self.capture_source not in { + "macos-screencapturekit", + "macos-screencapturekit-stream", + "macos-screencapture-utility", + }: + raise ValueError( + "an off-screen window requires a proven exact-window capture source" + ) + if self.capture_source == "macos-screencapturekit-stream": + if ( + self.frame_status is None + or self.frame_display_time is None + or self.pixel_display_time is None + or self.stream_generation is None + or self.stream_sequence is None + or self.capture_evidence_sha256 is None + ): + raise ValueError( + "a ScreenCaptureKit stream frame requires retained frame evidence" + ) + if self.pixel_display_time > self.frame_display_time: + raise ValueError( + "ScreenCaptureKit pixel time cannot follow its frame proof time" + ) + if ( + self.frame_status == "complete" + and self.pixel_display_time != self.frame_display_time + ): + raise ValueError( + "a complete ScreenCaptureKit frame must bind its pixel time" + ) + elif any( + value is not None + for value in ( + self.frame_status, + self.frame_display_time, + self.pixel_display_time, + self.stream_generation, + self.stream_sequence, + ) + ): + raise ValueError( + "non-stream capture sources cannot retain ScreenCaptureKit frame evidence" + ) if any(value <= 0 for value in (*self.viewport, *self.source_viewport)): raise ValueError("window capture viewports must be positive") left, top, width, height = self.content_rect @@ -162,6 +221,17 @@ def _closed_geometry(self) -> "WindowCaptureStateV2": self.model_dump(mode="json", exclude={"geometry_epoch_sha256"}) ): raise ValueError("window geometry epoch digest is invalid") + if ( + self.capture_evidence_sha256 is not None + and self.capture_evidence_sha256 + != window_capture_evidence_sha256( + self.model_dump( + mode="json", + exclude={"capture_evidence_sha256"}, + ) + ) + ): + raise ValueError("window capture evidence digest is invalid") return self diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index e244895..32195bc 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -80,6 +80,7 @@ WindowCapturePermissionError, WindowCaptureScope, build_window_scope, + prepare_macos_window_capture_runtime, ) CoordinateScope = WindowCaptureScope | DesktopCaptureScope @@ -2693,6 +2694,7 @@ def record( window_scope = build_window_scope( window_owner or config.RECORD_WINDOW_OWNER, window_title or config.RECORD_WINDOW_TITLE, + frame_rate=config.SCREEN_CAPTURE_FPS, ) initial_window_frame = None display_scope = DesktopCaptureScope.current() @@ -2703,9 +2705,14 @@ def record( display_scope.assert_current, ) initial_window_frame, _ = window_scope.capture_frame(publish=False) + window_snapshot = window_scope.snapshot() logger.info( - f"window-scoped capture resolved: {window_scope.snapshot()} " - f"initial frame {initial_window_frame.size}" + "window-scoped capture resolved: window_id={} provider={} " + "visibility_independent={} viewport={}", + window_snapshot.get("window_id"), + window_snapshot.get("capture_source"), + window_snapshot.get("visibility_independent"), + initial_window_frame.size, ) else: # MSS monitor zero is the exact combined frame read by @@ -3296,6 +3303,8 @@ def record( status_pipe.send({"type": "record.stopped"}) return event_q.last_source_ordinal if window_scope is not None else None finally: + if window_scope is not None: + window_scope.close() _force_reap_processes(task_by_name) _release_queues(writer_queues) @@ -3844,6 +3853,17 @@ def _run_record(self) -> None: self._finalized_event.set() def __enter__(self) -> "Recorder": + if ( + sys.platform == "darwin" + and ( + self._recording_config.window_owner + or self._recording_config.window_title + ) + ): + # Cocoa initialization must occur on the caller's main thread. + # The persistent ScreenCaptureKit stream starts later in the + # recorder worker thread. + prepare_macos_window_capture_runtime() if self._control_enabled: try: self._start_control_server() diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index ce26993..e9ad030 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -36,10 +36,12 @@ import hashlib import json import math +import os import subprocess import sys import tempfile import threading +import time from contextlib import contextmanager from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, Iterator, Optional @@ -88,6 +90,35 @@ def window_geometry_epoch_sha256(state: dict) -> str: return hashlib.sha256(encoded).hexdigest() +def window_capture_evidence_sha256(state: dict) -> str: + """Hash provider evidence without changing the stable geometry digest.""" + payload = { + key: state.get(key) + for key in ( + "geometry_epoch_sha256", + "capture_source", + "visibility_independent", + "on_screen", + "frame_status", + "frame_display_time", + "pixel_display_time", + "stream_generation", + "stream_sequence", + ) + } + encoded = json.dumps( + { + "schema_domain": "openadapt.capture.window-frame-evidence/v1", + **payload, + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + class WindowCaptureError(RuntimeError): """The target window could not be resolved or captured. @@ -152,7 +183,7 @@ def from_spec(cls, spec: "WindowTarget | dict | None") -> "WindowTarget | None": @dataclass(frozen=True) class TargetWindow: - """One resolved on-screen window. + """One resolved exact window. ``bounds`` is ``(x, y, w, h)`` in screen points, top-left origin — the same space as native global mouse coordinates. Same field semantics as @@ -223,11 +254,19 @@ def __init__( target: WindowTarget, resolver: ResolverFn | None = None, capturer: CapturerFn | None = None, + frame_rate: float | None = None, ) -> None: """Initialize the scope for ``target``.""" self.target = target self._resolver = resolver or resolve_window - self._capturer = capturer or capture_window + self._capture_provider = None + if capturer is None and sys.platform == "darwin": + self._capture_provider = _MacOSWindowCaptureProvider( + frame_rate=frame_rate, + ) + self._capturer = self._capture_provider.capture + else: + self._capturer = capturer or capture_window self._lock = threading.Lock() self._observation_lock = threading.RLock() self._window: TargetWindow | None = None @@ -239,6 +278,11 @@ def __init__( self._content_rect: tuple[int, int, int, int] | None = None self._fit_scale: float | None = None self._capture_source: str | None = None + self._frame_status: str | None = None + self._frame_display_time: int | None = None + self._pixel_display_time: int | None = None + self._stream_sequence: int | None = None + self._stream_generation: int | None = None self._geometry_generation = 0 self._geometry_signature: tuple | None = None self._published_generation = 0 @@ -255,6 +299,17 @@ def __init__( # first frame's timeline entry. self._frame_window: TargetWindow | None = None + def close(self) -> None: + """Release an owned platform capture provider. + + Injected capturers remain caller-owned. The macOS recorder owns one + persistent ScreenCaptureKit stream and must stop it before the session + reaches its terminal state. + """ + provider = self._capture_provider + if provider is not None: + provider.close() + @contextmanager def observation_boundary(self) -> Iterator[None]: """Serialize a frame acquisition with native input observation.""" @@ -408,6 +463,11 @@ def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: capture_source = str( source_image.info.get("openadapt_capture_source", win.capture_source) ) + frame_status = source_image.info.get("openadapt_frame_status") + frame_display_time = source_image.info.get("openadapt_frame_display_time") + pixel_display_time = source_image.info.get("openadapt_pixel_display_time") + stream_sequence = source_image.info.get("openadapt_stream_sequence") + stream_generation = source_image.info.get("openadapt_stream_generation") source_viewport = (source_image.width, source_image.height) output_viewport = output_viewport or source_viewport output_width, output_height = output_viewport @@ -460,6 +520,21 @@ def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: self._content_rect = (offset_x, offset_y, fitted_width, fitted_height) self._fit_scale = fit_scale self._capture_source = capture_source + self._frame_status = ( + str(frame_status) if frame_status is not None else None + ) + self._frame_display_time = ( + int(frame_display_time) if frame_display_time is not None else None + ) + self._pixel_display_time = ( + int(pixel_display_time) if pixel_display_time is not None else None + ) + self._stream_sequence = ( + int(stream_sequence) if stream_sequence is not None else None + ) + self._stream_generation = ( + int(stream_generation) if stream_generation is not None else None + ) if self._bound_identity is None: self._bound_identity = win.identity if geometry_signature != self._geometry_signature: @@ -638,6 +713,11 @@ def window_event_data(self) -> dict: content_rect = self._content_rect fit_scale = self._fit_scale capture_source = self._capture_source + frame_status = self._frame_status + frame_display_time = self._frame_display_time + pixel_display_time = self._pixel_display_time + stream_sequence = self._stream_sequence + stream_generation = self._stream_generation generation = self._geometry_generation topology = self._display_topology if window is None: @@ -653,6 +733,11 @@ def window_event_data(self) -> dict: "coordinate_source": window.coordinate_source, "capture_source": capture_source or window.capture_source, "visibility_independent": window.visibility_independent, + "frame_status": frame_status, + "frame_display_time": frame_display_time, + "pixel_display_time": pixel_display_time, + "stream_generation": stream_generation, + "stream_sequence": stream_sequence, "geometry_generation": generation, "display_topology_sha256": ( topology.get("topology_sha256") if topology else None @@ -668,6 +753,7 @@ def window_event_data(self) -> dict: "on_screen": window.on_screen, } state["geometry_epoch_sha256"] = window_geometry_epoch_sha256(state) + state["capture_evidence_sha256"] = window_capture_evidence_sha256(state) return { "title": window.title, "left": int(x), @@ -836,19 +922,63 @@ def _process_start_time(pid: int) -> float: _MACOS_CAPTURE_TIMEOUT_SECONDS = 15.0 -_MACOS_SC_WINDOW_CACHE: dict[int, object] = {} -_MACOS_SC_WINDOW_CACHE_LOCK = threading.Lock() +_MACOS_PROVIDER_CHAIN_TIMEOUT_SECONDS = 20.0 +_MACOS_SCK_ATTEMPT_TIMEOUT_SECONDS = 12.0 +_MACOS_RUNTIME_LOCK = threading.Lock() +_MACOS_RUNTIME_READY = False +_MACOS_SC_OUTPUT_CLASS: type | None = None +_MACOS_SC_OUTPUT_CLASS_LOCK = threading.Lock() + + +def prepare_macos_window_capture_runtime() -> None: + """Initialize AppKit before ScreenCaptureKit runs on a worker thread.""" + global _MACOS_RUNTIME_READY + if ( + sys.platform != "darwin" + or _MACOS_RUNTIME_READY + or not _screen_capture_kit_available() + ): + return + with _MACOS_RUNTIME_LOCK: + if _MACOS_RUNTIME_READY: + return + try: + import AppKit + + loaded = AppKit.NSApplicationLoad() + except (ImportError, OSError) as exc: + raise WindowCaptureUnavailableError( + "macOS window capture could not initialize AppKit" + ) from exc + if loaded is False: + raise WindowCaptureUnavailableError( + "macOS window capture could not load AppKit" + ) + _MACOS_RUNTIME_READY = True def _screen_capture_kit_available() -> bool: - """Return whether single-frame desktop-independent capture is available.""" + """Return whether persistent desktop-independent capture is available.""" try: import ScreenCaptureKit except ImportError: return False - return hasattr( - ScreenCaptureKit.SCScreenshotManager, - "captureImageWithFilter_configuration_completionHandler_", + return all( + hasattr(ScreenCaptureKit, name) + for name in ( + "SCContentFilter", + "SCShareableContent", + "SCStream", + "SCStreamConfiguration", + ) + ) + + +def _macos_visibility_independent_capture_available() -> bool: + """Return whether an exact window can be captured without visibility.""" + return _screen_capture_kit_available() or ( + os.path.isfile("/usr/sbin/screencapture") + and os.access("/usr/sbin/screencapture", os.X_OK) ) @@ -856,6 +986,7 @@ def _macos_completion( start: Callable[[Callable[..., None]], None], *, operation: str, + timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS, ) -> object: """Wait for one ScreenCaptureKit completion without requiring an event loop.""" completed = threading.Event() @@ -872,7 +1003,7 @@ def completion(value: object | None, error: object | None) -> None: raise WindowCaptureUnavailableError( f"ScreenCaptureKit could not start {operation}" ) from exc - if not completed.wait(_MACOS_CAPTURE_TIMEOUT_SECONDS): + if not completed.wait(max(0.001, timeout_seconds)): raise WindowCaptureUnavailableError( f"ScreenCaptureKit timed out during {operation}" ) @@ -896,13 +1027,51 @@ def completion(value: object | None, error: object | None) -> None: return result["value"] -def _screen_capture_kit_window(window_id: int) -> object: - """Return the exact shareable window, including windows on other Spaces.""" - with _MACOS_SC_WINDOW_CACHE_LOCK: - cached = _MACOS_SC_WINDOW_CACHE.get(window_id) - if cached is not None: - return cached +def _macos_error_completion( + start: Callable[[Callable[[object | None], None]], None], + *, + operation: str, + timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS, +) -> None: + """Wait for a ScreenCaptureKit start/stop completion callback.""" + completed = threading.Event() + result: dict[str, object | None] = {"error": None} + + def completion(error: object | None) -> None: + result["error"] = error + completed.set() + + try: + start(completion) + except Exception as exc: + raise WindowCaptureUnavailableError( + f"ScreenCaptureKit could not start {operation}" + ) from exc + if not completed.wait(max(0.001, timeout_seconds)): + raise WindowCaptureUnavailableError( + f"ScreenCaptureKit timed out during {operation}" + ) + if result["error"] is not None: + error = result["error"] + code_getter = getattr(error, "code", None) + code = int(code_getter()) if callable(code_getter) else None + error_type = ( + WindowCapturePermissionError + if code in {-3801, -3803} + else WindowCaptureError + ) + raise error_type( + f"ScreenCaptureKit failed during {operation}" + + (f" (error {code})" if code is not None else "") + ) + +def _screen_capture_kit_window( + window_id: int, + *, + timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS, +) -> object: + """Return the exact shareable window, including windows on other Spaces.""" try: import ScreenCaptureKit except ImportError as exc: @@ -921,6 +1090,7 @@ def _screen_capture_kit_window(window_id: int) -> object: callback, ), operation="window enumeration", + timeout_seconds=timeout_seconds, ) windows = list(content.windows() or []) match = next( @@ -931,8 +1101,6 @@ def _screen_capture_kit_window(window_id: int) -> object: raise WindowCaptureError( f"ScreenCaptureKit cannot access exact window {window_id}" ) - with _MACOS_SC_WINDOW_CACHE_LOCK: - _MACOS_SC_WINDOW_CACHE[window_id] = match return match @@ -961,44 +1129,434 @@ def _pil_image_from_cgimage(img_ref: object, *, source: str) -> "Image.Image": return image -def _capture_window_macos_screencapturekit(window: TargetWindow) -> "Image.Image": - """Capture one exact window without requiring it to be frontmost or visible.""" +def _screen_capture_kit_output_class() -> type: + """Build the Objective-C stream delegate once, without display access.""" + global _MACOS_SC_OUTPUT_CLASS + if _MACOS_SC_OUTPUT_CLASS is not None: + return _MACOS_SC_OUTPUT_CLASS + with _MACOS_SC_OUTPUT_CLASS_LOCK: + if _MACOS_SC_OUTPUT_CLASS is not None: + return _MACOS_SC_OUTPUT_CLASS + try: + import objc + from Foundation import NSObject + except ImportError as exc: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit requires PyObjC Cocoa bindings" + ) from exc + + class OpenAdaptSCStreamOutput( + NSObject, + protocols=[ + objc.protocolNamed("SCStreamOutput"), + objc.protocolNamed("SCStreamDelegate"), + ], + ): + def initWithOwner_(self, owner): + self = objc.super(OpenAdaptSCStreamOutput, self).init() + if self is not None: + self._openadapt_owner = owner + return self + + @objc.typedSelector(b"v@:@^{opaqueCMSampleBuffer=}q") + def stream_didOutputSampleBuffer_ofType_( + self, + _stream, + sample_buffer, + output_type, + ): + self._openadapt_owner._receive_sample( + self._openadapt_generation, + sample_buffer, + output_type, + ) + + @objc.typedSelector(b"v@:@@") + def stream_didStopWithError_(self, _stream, error): + self._openadapt_owner._receive_stop( + self._openadapt_generation, + error, + ) + + _MACOS_SC_OUTPUT_CLASS = OpenAdaptSCStreamOutput + return _MACOS_SC_OUTPUT_CLASS + + +def _macos_dispatch_queue(label: bytes) -> object: + """Create the serial dispatch queue required by SCStreamOutput.""" try: - import ScreenCaptureKit - except ImportError as exc: + import objc + + functions: dict[str, object] = {} + objc.loadBundleFunctions(None, functions, [("dispatch_queue_create", b"@*@")]) + return functions["dispatch_queue_create"](label, None) + except Exception as exc: raise WindowCaptureUnavailableError( - "ScreenCaptureKit requires pyobjc-framework-ScreenCaptureKit" + "ScreenCaptureKit could not create its callback queue" ) from exc - sc_window = _screen_capture_kit_window(window.window_id) - content_filter = ScreenCaptureKit.SCContentFilter.alloc().initWithDesktopIndependentWindow_( - sc_window - ) - point_scale = float(content_filter.pointPixelScale()) - if not math.isfinite(point_scale) or point_scale <= 0: - raise WindowCaptureError("ScreenCaptureKit returned an invalid point-to-pixel scale") - configuration = ScreenCaptureKit.SCStreamConfiguration.alloc().init() - configuration.setWidth_(max(1, round(window.bounds[2] * point_scale))) - configuration.setHeight_(max(1, round(window.bounds[3] * point_scale))) - configuration.setShowsCursor_(False) - if hasattr(configuration, "setIgnoreShadowsSingleWindow_"): - configuration.setIgnoreShadowsSingleWindow_(True) - capture_image = getattr( - ScreenCaptureKit.SCScreenshotManager, - "captureImageWithFilter_configuration_completionHandler_", + +def _pil_image_from_sample_buffer(sample_buffer: object) -> "Image.Image": + """Copy a ScreenCaptureKit sample buffer into stable RGB pixels.""" + try: + import CoreMedia + import Quartz + + pixel_buffer = CoreMedia.CMSampleBufferGetImageBuffer(sample_buffer) + if pixel_buffer is None: + raise WindowCaptureError("ScreenCaptureKit returned no pixel buffer") + ci_image = Quartz.CIImage.imageWithCVPixelBuffer_(pixel_buffer) + context = Quartz.CIContext.contextWithOptions_(None) + img_ref = context.createCGImage_fromRect_(ci_image, ci_image.extent()) + except WindowCaptureError: + raise + except Exception as exc: + raise WindowCaptureError( + "ScreenCaptureKit returned an unreadable pixel buffer" + ) from exc + if img_ref is None: + raise WindowCaptureError("ScreenCaptureKit returned no window image") + return _pil_image_from_cgimage( + img_ref, + source="macos-screencapturekit-stream", ) - img_ref = _macos_completion( - lambda callback: capture_image( + + +class _MacOSScreenCaptureKitStream: + """One persistent desktop-independent stream for one exact window ID.""" + + def __init__(self, *, frame_rate: float | None = None) -> None: + self._condition = threading.Condition() + self._capture_lock = threading.RLock() + self._stream = None + self._delegate = None + self._queue = None + self._window_id: int | None = None + self._bounds_size: tuple[float, float] | None = None + self._sequence = 0 + self._delivered_sequence = 0 + self._last_complete_image = None + self._last_complete_display_time: int | None = None + self._last_display_time: int | None = None + self._last_status: int | None = None + self._error: WindowCaptureError | None = None + self._closing = False + self._closed = False + self._generation = 0 + self._frame_rate = ( + float(frame_rate) + if frame_rate is not None + and math.isfinite(float(frame_rate)) + and float(frame_rate) > 0 + else 10.0 + ) + + def _receive_sample( + self, + generation: int, + sample_buffer: object, + output_type: int, + ) -> None: + with self._condition: + if generation != self._generation or self._closing or self._closed: + return + try: + import CoreMedia + import ScreenCaptureKit + + if int(output_type) != int(ScreenCaptureKit.SCStreamOutputTypeScreen): + return + attachments = CoreMedia.CMSampleBufferGetSampleAttachmentsArray( + sample_buffer, + False, + ) + attachment = list(attachments or [])[0] if attachments else {} + status_value = attachment.get(ScreenCaptureKit.SCStreamFrameInfoStatus) + if status_value is None: + raise WindowCaptureError( + "ScreenCaptureKit frame has no explicit status" + ) + status = int(status_value) + display_time_value = attachment.get( + ScreenCaptureKit.SCStreamFrameInfoDisplayTime + ) + if display_time_value is None: + raise WindowCaptureError( + "ScreenCaptureKit frame has no display time" + ) + display_time = int(display_time_value) + image = None + error = None + if status == int(ScreenCaptureKit.SCFrameStatusComplete): + image = _pil_image_from_sample_buffer(sample_buffer) + elif status not in ( + int(ScreenCaptureKit.SCFrameStatusIdle), + int(ScreenCaptureKit.SCFrameStatusStarted), + ): + error = WindowCaptureError( + f"ScreenCaptureKit returned unusable frame status {status}" + ) + except Exception as exc: + image = None + status = -1 + display_time = None + error = ( + exc + if isinstance(exc, WindowCaptureError) + else WindowCaptureError("ScreenCaptureKit frame processing failed") + ) + with self._condition: + if generation != self._generation or self._closing or self._closed: + return + self._sequence += 1 + self._last_status = status + if display_time is not None: + self._last_display_time = display_time + if image is not None: + self._last_complete_image = image + self._last_complete_display_time = display_time + if error is not None: + self._error = error + self._condition.notify_all() + + def _receive_stop(self, generation: int, error: object | None) -> None: + with self._condition: + if generation != self._generation: + return + if not self._closing: + code_getter = getattr(error, "code", None) + code = int(code_getter()) if callable(code_getter) else None + self._error = WindowCaptureError( + "ScreenCaptureKit stopped the exact-window stream" + + (f" (error {code})" if code is not None else "") + ) + self._condition.notify_all() + + @staticmethod + def _remaining(deadline: float, operation: str) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise WindowCaptureUnavailableError( + f"ScreenCaptureKit timed out before {operation}" + ) + return remaining + + def _start(self, window: TargetWindow, *, deadline: float) -> None: + try: + import CoreMedia + import ScreenCaptureKit + except ImportError as exc: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit requires its PyObjC framework bindings" + ) from exc + + prepare_macos_window_capture_runtime() + sc_window = _screen_capture_kit_window( + window.window_id, + timeout_seconds=self._remaining(deadline, "window enumeration"), + ) + content_filter = ( + ScreenCaptureKit.SCContentFilter.alloc().initWithDesktopIndependentWindow_( + sc_window + ) + ) + point_scale = float(content_filter.pointPixelScale()) + if not math.isfinite(point_scale) or point_scale <= 0: + raise WindowCaptureError( + "ScreenCaptureKit returned an invalid point-to-pixel scale" + ) + configuration = ScreenCaptureKit.SCStreamConfiguration.alloc().init() + configuration.setWidth_(max(1, round(window.bounds[2] * point_scale))) + configuration.setHeight_(max(1, round(window.bounds[3] * point_scale))) + configuration.setShowsCursor_(False) + configuration.setQueueDepth_(3) + configuration.setMinimumFrameInterval_( + CoreMedia.CMTimeMakeWithSeconds(1.0 / self._frame_rate, 600) + ) + if hasattr(configuration, "setIgnoreShadowsSingleWindow_"): + configuration.setIgnoreShadowsSingleWindow_(True) + + with self._condition: + if self._closed: + raise WindowCaptureError("ScreenCaptureKit stream is closed") + self._generation += 1 + generation = self._generation + output_class = _screen_capture_kit_output_class() + delegate = output_class.alloc().initWithOwner_(self) + delegate._openadapt_generation = generation + queue = _macos_dispatch_queue( + f"ai.openadapt.capture.window.{window.window_id}".encode("ascii") + ) + stream = ScreenCaptureKit.SCStream.alloc().initWithFilter_configuration_delegate_( content_filter, configuration, - callback, - ), - operation=f"exact-window capture for window {window.window_id}", - ) - return _pil_image_from_cgimage(img_ref, source="macos-screencapturekit") + delegate, + ) + result = stream.addStreamOutput_type_sampleHandlerQueue_error_( + delegate, + ScreenCaptureKit.SCStreamOutputTypeScreen, + queue, + None, + ) + if isinstance(result, tuple): + added, add_error = result + else: + added, add_error = result, None + if not added: + code_getter = getattr(add_error, "code", None) + code = int(code_getter()) if callable(code_getter) else None + raise WindowCaptureError( + "ScreenCaptureKit could not add the exact-window output" + + (f" (error {code})" if code is not None else "") + ) + with self._condition: + self._stream = stream + self._delegate = delegate + self._queue = queue + self._window_id = window.window_id + self._bounds_size = (window.bounds[2], window.bounds[3]) + self._sequence = 0 + self._delivered_sequence = 0 + self._last_complete_image = None + self._last_complete_display_time = None + self._last_display_time = None + self._last_status = None + self._error = None + self._closing = False + try: + _macos_error_completion( + lambda callback: stream.startCaptureWithCompletionHandler_(callback), + operation=f"exact-window stream start for window {window.window_id}", + timeout_seconds=self._remaining(deadline, "stream start"), + ) + except Exception: + self._stop_stream(suppress_errors=True, deadline=deadline) + raise + + def capture( + self, + window: TargetWindow, + *, + deadline: float | None = None, + ) -> "Image.Image": + """Return the next complete or proven-idle frame from the stream.""" + try: + import ScreenCaptureKit + except ImportError as exc: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit requires its PyObjC framework bindings" + ) from exc + usable_statuses = { + int(ScreenCaptureKit.SCFrameStatusComplete), + int(ScreenCaptureKit.SCFrameStatusIdle), + } + deadline = deadline or ( + time.monotonic() + _MACOS_SCK_ATTEMPT_TIMEOUT_SECONDS + ) + with self._capture_lock: + if self._closed: + raise WindowCaptureError("ScreenCaptureKit stream is closed") + desired_size = (window.bounds[2], window.bounds[3]) + if ( + self._stream is None + or self._window_id != window.window_id + or self._bounds_size != desired_size + ): + self._stop_stream(suppress_errors=False, deadline=deadline) + self._start(window, deadline=deadline) + with self._condition: + while ( + ( + self._sequence <= self._delivered_sequence + or self._last_complete_image is None + or self._last_status not in usable_statuses + ) + and self._error is None + ): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise WindowCaptureUnavailableError( + "ScreenCaptureKit timed out waiting for an exact-window frame" + ) + self._condition.wait(remaining) + if self._error is not None: + raise self._error + self._delivered_sequence = self._sequence + image = self._last_complete_image.copy() + image.info["openadapt_capture_source"] = ( + "macos-screencapturekit-stream" + ) + image.info["openadapt_frame_status"] = ( + "complete" + if self._last_status + == int(ScreenCaptureKit.SCFrameStatusComplete) + else "idle" + ) + image.info["openadapt_frame_display_time"] = self._last_display_time + image.info["openadapt_pixel_display_time"] = ( + self._last_complete_display_time + ) + image.info["openadapt_stream_sequence"] = self._sequence + image.info["openadapt_stream_generation"] = self._generation + return image + + def _stop_stream( + self, + *, + suppress_errors: bool, + deadline: float | None = None, + ) -> None: + """Stop the current generation. The caller holds ``_capture_lock``.""" + with self._condition: + stream = self._stream + self._closing = True + stop_error = None + if stream is not None: + try: + _macos_error_completion( + lambda callback: stream.stopCaptureWithCompletionHandler_(callback), + operation="exact-window stream stop", + timeout_seconds=( + self._remaining(deadline, "stream stop") + if deadline is not None + else _MACOS_CAPTURE_TIMEOUT_SECONDS + ), + ) + except WindowCaptureError as exc: + stop_error = exc + with self._condition: + self._stream = None + self._delegate = None + self._queue = None + self._window_id = None + self._bounds_size = None + if not self._closed: + self._closing = False + self._condition.notify_all() + if stop_error is not None: + if suppress_errors: + logger.warning("ScreenCaptureKit stream stop failed: {}", stop_error) + else: + raise stop_error + + def close(self, *, timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS) -> None: + """Stop the stream and wake any waiting capture.""" + with self._condition: + self._closed = True + self._error = WindowCaptureError("ScreenCaptureKit stream was closed") + self._condition.notify_all() + with self._capture_lock: + self._stop_stream( + suppress_errors=True, + deadline=time.monotonic() + max(0.001, timeout_seconds), + ) -def _capture_window_macos_utility(window: TargetWindow) -> "Image.Image": +def _capture_window_macos_utility( + window: TargetWindow, + *, + timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS, +) -> "Image.Image": """Use the signed system utility as an exact-window compatibility path.""" from PIL import Image @@ -1017,7 +1575,7 @@ def _capture_window_macos_utility(window: TargetWindow) -> "Image.Image": check=False, capture_output=True, text=True, - timeout=_MACOS_CAPTURE_TIMEOUT_SECONDS, + timeout=max(0.001, timeout_seconds), ) except (OSError, subprocess.TimeoutExpired) as exc: raise WindowCaptureUnavailableError( @@ -1046,7 +1604,7 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: """ import Quartz - visibility_independent = _screen_capture_kit_available() + visibility_independent = _macos_visibility_independent_capture_available() owner_l = target.owner.lower() if target.owner else None title_l = target.title.lower() if target.title else None wins = Quartz.CGWindowListCopyWindowInfo(Quartz.kCGWindowListOptionAll, Quartz.kCGNullWindowID) @@ -1116,54 +1674,124 @@ def _capture_window_macos(window: TargetWindow) -> "Image.Image": system utility is the final exact-window compatibility path. No provider can substitute a full-screen image or another window. """ - import Quartz + provider = _MacOSWindowCaptureProvider() + try: + return provider.capture(window) + finally: + provider.close() - failures: list[BaseException] = [] - if _screen_capture_kit_available(): - try: - return _capture_window_macos_screencapturekit(window) - except WindowCaptureError as exc: - failures.append(exc) + +class _MacOSWindowCaptureProvider: + """Session-owned exact-window providers with sticky safe fallback.""" + + def __init__(self, *, frame_rate: float | None = None) -> None: + self._stream = ( + _MacOSScreenCaptureKitStream(frame_rate=frame_rate) + if _screen_capture_kit_available() + else None + ) + self._sck_disabled = self._stream is None + self._quartz_disabled = False + + def capture(self, window: TargetWindow) -> "Image.Image": + """Capture only ``window`` and never substitute desktop pixels.""" + failures: list[BaseException] = [] + chain_deadline = time.monotonic() + _MACOS_PROVIDER_CHAIN_TIMEOUT_SECONDS + if not self._sck_disabled and self._stream is not None: + try: + return self._stream.capture( + window, + deadline=min( + chain_deadline, + time.monotonic() + _MACOS_SCK_ATTEMPT_TIMEOUT_SECONDS, + ), + ) + except WindowCaptureError as exc: + failures.append(exc) + self._stream.close(timeout_seconds=2.0) + self._sck_disabled = True + logger.warning( + "ScreenCaptureKit exact-window stream failed; disabling it " + "for this recording session" + ) + + if window.on_screen and not self._quartz_disabled: + try: + import Quartz + + img_ref = Quartz.CGWindowListCreateImage( + Quartz.CGRectNull, + Quartz.kCGWindowListOptionIncludingWindow, + window.window_id, + Quartz.kCGWindowImageBoundsIgnoreFraming, + ) + except Exception as exc: + img_ref = None + failures.append( + WindowCaptureUnavailableError( + "Quartz exact-window capture could not run" + ) + ) + logger.debug("Quartz exact-window capture failed: {}", exc) + if img_ref is not None: + return _pil_image_from_cgimage( + img_ref, + source="macos-quartz-window-image", + ) + self._quartz_disabled = True + failures.append( + WindowCapturePermissionError( + f"Quartz returned no image for exact window {window.window_id}" + ) + ) logger.warning( - "ScreenCaptureKit exact-window capture failed; trying the " - "legacy exact-window providers" + "Quartz exact-window capture returned no image; disabling it " + "for this recording session" + ) + elif not window.on_screen: + failures.append( + WindowCapturePermissionError( + "Quartz was skipped because the exact target is not on screen" + ) ) - img_ref = None - if window.on_screen: - img_ref = Quartz.CGWindowListCreateImage( - Quartz.CGRectNull, - Quartz.kCGWindowListOptionIncludingWindow, - window.window_id, - Quartz.kCGWindowImageBoundsIgnoreFraming, - ) - if img_ref is not None: - return _pil_image_from_cgimage(img_ref, source="macos-quartz-window-image") - failures.append( - WindowCapturePermissionError( - ( - "Quartz returned no image" - if window.on_screen - else "Quartz was skipped because the target is not on screen" + try: + remaining = chain_deadline - time.monotonic() + if remaining <= 0: + raise WindowCaptureUnavailableError( + "the exact-window provider chain exhausted its startup budget" + ) + return _capture_window_macos_utility( + window, + timeout_seconds=min(_MACOS_CAPTURE_TIMEOUT_SECONDS, remaining), ) - + f" for exact window {window.window_id}" - ) - ) - try: - return _capture_window_macos_utility(window) - except WindowCaptureError as exc: - failures.append(exc) - failure = WindowCaptureError( - f"all exact-window capture providers failed for window {window.window_id}" - ) - for provider_failure in failures: - try: - failure.add_note( - f"{type(provider_failure).__name__}: {provider_failure}" + except WindowCaptureError as exc: + failures.append(exc) + failure_type = ( + WindowCapturePermissionError + if failures + and all( + isinstance(item, WindowCapturePermissionError) + for item in failures ) - except AttributeError: # Python 3.10 - pass - raise failure from exc + else WindowCaptureError + ) + failure = failure_type( + f"all exact-window capture providers failed for window {window.window_id}" + ) + for provider_failure in failures: + try: + failure.add_note( + f"{type(provider_failure).__name__}: {provider_failure}" + ) + except AttributeError: # Python 3.10 + pass + raise failure from exc + + def close(self) -> None: + """Stop the owned stream, if it started.""" + if self._stream is not None: + self._stream.close() def _resolve_window_windows(target: WindowTarget) -> TargetWindow | None: @@ -1284,7 +1912,12 @@ def _capture_window_windows(window: TargetWindow) -> "Image.Image": return Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") -def build_window_scope(owner: str | None, title: str | None) -> WindowCaptureScope | None: +def build_window_scope( + owner: str | None, + title: str | None, + *, + frame_rate: float | None = None, +) -> WindowCaptureScope | None: """Build a :class:`WindowCaptureScope` when a target is configured. Central place the recorder uses to turn (possibly-empty) config values @@ -1292,5 +1925,12 @@ def build_window_scope(owner: str | None, title: str | None) -> WindowCaptureSco """ if not (owner or title): return None - logger.info(f"window-scoped capture: owner={owner!r} title={title!r}") - return WindowCaptureScope(WindowTarget(owner=owner, title=title)) + logger.info( + "window-scoped capture configured: owner_selector={} title_selector={}", + bool(owner), + bool(title), + ) + return WindowCaptureScope( + WindowTarget(owner=owner, title=title), + frame_rate=frame_rate, + ) diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 1e8831b..cca7b6b 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -912,8 +912,52 @@ def test_window_capture_state_rejects_scales_not_derived_from_content(scope): WindowCaptureStateV2.model_validate(state) +def test_window_capture_state_accepts_proven_offscreen_sck_frame(scope): + scope.capture_frame() + state = scope.window_event_data()["state"] + state.update( + { + "on_screen": False, + "visibility_independent": True, + "capture_source": "macos-screencapturekit-stream", + "frame_status": "complete", + "frame_display_time": 100, + "pixel_display_time": 100, + "stream_generation": 1, + "stream_sequence": 1, + } + ) + state["capture_evidence_sha256"] = ( + window_capture_module.window_capture_evidence_sha256(state) + ) + + parsed = WindowCaptureStateV2.model_validate(state) + + assert parsed.on_screen is False + assert parsed.visibility_independent is True + + +def test_window_capture_state_rejects_unproven_offscreen_frame(scope): + scope.capture_frame() + state = scope.window_event_data()["state"] + state.update( + { + "on_screen": False, + "visibility_independent": True, + "capture_source": "macos-quartz-window-image", + } + ) + + with pytest.raises(ValueError, match="proven exact-window capture source"): + WindowCaptureStateV2.model_validate(state) + + def test_macos_resolver_ignores_a_larger_hidden_matching_window(monkeypatch): - monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr( + window_capture_module, + "_macos_visibility_independent_capture_available", + lambda: False, + ) hidden = { "kCGWindowOwnerName": "FakeApp", "kCGWindowName": "Document", @@ -977,6 +1021,36 @@ def test_macos_resolver_accepts_offspace_window_with_screencapturekit(monkeypatc assert resolved.capture_source == "macos-exact-window-provider-chain" +def test_macos_resolver_accepts_offspace_window_with_exact_utility(monkeypatch): + hidden = { + "kCGWindowOwnerName": "FakeApp", + "kCGWindowName": "Document", + "kCGWindowLayer": 0, + "kCGWindowIsOnscreen": False, + "kCGWindowBounds": {"X": 0, "Y": 0, "Width": 1512, "Height": 944}, + "kCGWindowOwnerPID": 100, + "kCGWindowNumber": 19373, + } + quartz = SimpleNamespace( + kCGWindowListOptionAll=1, + kCGNullWindowID=0, + CGWindowListCopyWindowInfo=lambda *_args: [hidden], + ) + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_process_start_time", lambda _pid: 123.0) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr(window_capture_module.os.path, "isfile", lambda _path: True) + monkeypatch.setattr(window_capture_module.os, "access", lambda *_args: True) + + resolved = window_capture_module._resolve_window_macos( + WindowTarget(owner="FakeApp", title="Document") + ) + + assert resolved is not None + assert resolved.on_screen is False + assert resolved.visibility_independent is True + + def test_macos_resolver_refuses_ambiguous_owner_only_match(monkeypatch): windows = [ { @@ -1018,24 +1092,122 @@ def test_macos_capture_uses_exact_utility_after_sck_and_quartz_fail(monkeypatch) ) quartz = SimpleNamespace() monkeypatch.setitem(sys.modules, "Quartz", quartz) + stream_calls = [] + + class DeniedStream: + def __init__(self, *, frame_rate=None): + assert frame_rate is None + + def capture(self, _window, **_kwargs): + stream_calls.append("capture") + raise WindowCapturePermissionError("denied") + + def close(self, **_kwargs): + stream_calls.append("close") + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) monkeypatch.setattr( window_capture_module, - "_capture_window_macos_screencapturekit", - lambda _window: (_ for _ in ()).throw(WindowCapturePermissionError("denied")), + "_MacOSScreenCaptureKitStream", + DeniedStream, ) expected = Image.new("RGB", (3024, 1888), "white") expected.info["openadapt_capture_source"] = "macos-screencapture-utility" monkeypatch.setattr( window_capture_module, "_capture_window_macos_utility", - lambda _window: expected, + lambda _window, **_kwargs: expected, ) captured = window_capture_module._capture_window_macos(window) assert captured.size == (3024, 1888) assert captured.info["openadapt_capture_source"] == "macos-screencapture-utility" + assert stream_calls == ["capture", "close", "close"] + + +def test_macos_provider_disables_failed_stream_for_the_session(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + stream_calls = [] + + class FailedStream: + def __init__(self, *, frame_rate=None): + assert frame_rate is None + + def capture(self, _window, **_kwargs): + stream_calls.append("capture") + raise WindowCapturePermissionError("denied") + + def close(self, **_kwargs): + stream_calls.append("close") + + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_MacOSScreenCaptureKitStream", + FailedStream, + ) + utility_calls = [] + + def utility(_window, **_kwargs): + utility_calls.append("capture") + return Image.new("RGB", (3024, 1888), "white") + + monkeypatch.setattr(window_capture_module, "_capture_window_macos_utility", utility) + provider = window_capture_module._MacOSWindowCaptureProvider() + + provider.capture(window) + provider.capture(window) + + assert stream_calls == ["capture", "close"] + assert utility_calls == ["capture", "capture"] + + +def test_macos_provider_disables_failed_quartz_for_the_session(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=True, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + quartz_calls = [] + quartz = SimpleNamespace( + CGRectNull=None, + kCGWindowListOptionIncludingWindow=1, + kCGWindowImageBoundsIgnoreFraming=2, + CGWindowListCreateImage=lambda *_args: quartz_calls.append("capture"), + ) + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + utility_calls = [] + + def utility(_window, **_kwargs): + utility_calls.append("capture") + return Image.new("RGB", (3024, 1888), "white") + + monkeypatch.setattr(window_capture_module, "_capture_window_macos_utility", utility) + provider = window_capture_module._MacOSWindowCaptureProvider() + + provider.capture(window) + provider.capture(window) + + assert quartz_calls == ["capture"] + assert utility_calls == ["capture", "capture"] def test_screencapturekit_enumerates_other_spaces_and_selects_exact_id(monkeypatch): @@ -1061,7 +1233,6 @@ def getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHan fake_sck = SimpleNamespace(SCShareableContent=FakeShareableContent) monkeypatch.setitem(sys.modules, "ScreenCaptureKit", fake_sck) - window_capture_module._MACOS_SC_WINDOW_CACHE.clear() resolved = window_capture_module._screen_capture_kit_window(19373) @@ -1069,6 +1240,173 @@ def getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHan assert calls == [(True, False)] +@pytest.mark.skipif(sys.platform != "darwin", reason="PyObjC protocol test") +def test_screencapturekit_output_class_has_native_protocol_signatures(): + if not window_capture_module._screen_capture_kit_available(): + pytest.skip("ScreenCaptureKit bindings are not installed") + + output_class = window_capture_module._screen_capture_kit_output_class() + + assert output_class is not None + + +def _fake_screencapturekit_modules(monkeypatch, attachment): + fake_sck = SimpleNamespace( + SCStreamOutputTypeScreen=0, + SCStreamFrameInfoStatus="status", + SCStreamFrameInfoDisplayTime="display_time", + SCFrameStatusComplete=0, + SCFrameStatusIdle=1, + SCFrameStatusStarted=2, + ) + fake_core_media = SimpleNamespace( + CMSampleBufferGetSampleAttachmentsArray=lambda *_args: [attachment], + ) + monkeypatch.setitem(sys.modules, "ScreenCaptureKit", fake_sck) + monkeypatch.setitem(sys.modules, "CoreMedia", fake_core_media) + + +def test_screencapturekit_stream_retains_complete_and_idle_evidence(monkeypatch): + _fake_screencapturekit_modules( + monkeypatch, + {"status": 0, "display_time": 100}, + ) + first = Image.new("RGB", (20, 10), "blue") + monkeypatch.setattr( + window_capture_module, + "_pil_image_from_sample_buffer", + lambda _sample: first, + ) + stream = window_capture_module._MacOSScreenCaptureKitStream(frame_rate=2.0) + stream._generation = 1 + + stream._receive_sample(1, object(), 0) + + assert stream._last_complete_image is first + assert stream._last_complete_display_time == 100 + assert stream._last_status == 0 + + sys.modules["CoreMedia"].CMSampleBufferGetSampleAttachmentsArray = ( + lambda *_args: [{"status": 1, "display_time": 120}] + ) + stream._receive_sample(1, object(), 0) + assert stream._last_complete_image is first + assert stream._last_complete_display_time == 100 + assert stream._last_display_time == 120 + assert stream._last_status == 1 + + +def test_screencapturekit_stream_rejects_missing_frame_metadata(monkeypatch): + _fake_screencapturekit_modules(monkeypatch, {}) + stream = window_capture_module._MacOSScreenCaptureKitStream() + stream._generation = 1 + + stream._receive_sample(1, object(), 0) + + assert isinstance(stream._error, WindowCaptureError) + assert "explicit status" in str(stream._error) + + +def test_screencapturekit_stream_ignores_late_generation(monkeypatch): + _fake_screencapturekit_modules( + monkeypatch, + {"status": 0, "display_time": 100}, + ) + monkeypatch.setattr( + window_capture_module, + "_pil_image_from_sample_buffer", + lambda _sample: Image.new("RGB", (20, 10), "blue"), + ) + stream = window_capture_module._MacOSScreenCaptureKitStream() + stream._generation = 2 + + stream._receive_sample(1, object(), 0) + + assert stream._sequence == 0 + assert stream._last_complete_image is None + + +def test_screencapturekit_close_wakes_waiting_capture(monkeypatch): + _fake_screencapturekit_modules(monkeypatch, {}) + + class FakeNativeStream: + @staticmethod + def stopCaptureWithCompletionHandler_(callback): + callback(None) + + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 20.0, 10.0), + process_start_time=123.0, + ) + stream = window_capture_module._MacOSScreenCaptureKitStream() + stream._stream = FakeNativeStream() + stream._window_id = window.window_id + stream._bounds_size = (window.bounds[2], window.bounds[3]) + stream._generation = 1 + errors = [] + + def wait_for_frame(): + try: + stream.capture(window, deadline=time.monotonic() + 5) + except WindowCaptureError as exc: + errors.append(exc) + + capture_thread = threading.Thread(target=wait_for_frame) + capture_thread.start() + time.sleep(0.02) + stream.close(timeout_seconds=0.5) + capture_thread.join(timeout=1) + + assert not capture_thread.is_alive() + assert len(errors) == 1 + assert "closed" in str(errors[0]) + + +def test_macos_provider_preserves_all_provider_permission_denial(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + + class DeniedStream: + def __init__(self, **_kwargs): + pass + + def capture(self, _window, **_kwargs): + raise WindowCapturePermissionError("denied") + + def close(self, **_kwargs): + pass + + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_MacOSScreenCaptureKitStream", + DeniedStream, + ) + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + WindowCapturePermissionError("denied") + ), + ) + + with pytest.raises(WindowCapturePermissionError, match="all exact-window"): + window_capture_module._MacOSWindowCaptureProvider().capture(window) + + class TestTranslatePoint: """Coordinate translation: global screen points -> window pixels.""" @@ -1842,25 +2180,27 @@ def _temporary_window_geometry(window: TargetWindow): class TestWindowCaptureLive: """Capture a real window end to end (resolve -> frame -> translate).""" - def _scope(self) -> WindowCaptureScope: + def _scope(self, *, require_on_screen: bool = False) -> WindowCaptureScope: scope = WindowCaptureScope(WindowTarget(owner=_SMOKE_OWNER, title=_SMOKE_TITLE)) try: resolved = scope.resolve() except WindowCaptureError as exc: if _PRODUCTION_QUALIFICATION: raise AssertionError( - f"production qualification requires an on-screen window " + f"production qualification requires an exact window " f"matching owner {_SMOKE_OWNER!r} title {_SMOKE_TITLE!r}" ) from exc pytest.skip( - f"no on-screen window matching owner {_SMOKE_OWNER!r} " + f"no exact window matching owner {_SMOKE_OWNER!r} " f"title {_SMOKE_TITLE!r} on this desktop; open one (or set " "OPENADAPT_WINDOW_SMOKE_OWNER) to run the live smoke test" ) - if not resolved.on_screen: + if require_on_screen and not resolved.on_screen: if _PRODUCTION_QUALIFICATION: - raise AssertionError("the production qualification window is not on screen") - pytest.skip("the matching live smoke-test window is not on screen") + raise AssertionError( + "the live geometry qualification window is not on screen" + ) + pytest.skip("the matching live geometry-test window is not on screen") desktop = DesktopCaptureScope.current() scope.bind_display_topology(desktop.snapshot(), desktop.assert_current) return scope @@ -1886,6 +2226,7 @@ def test_live_window_frame_and_translation(self): # Bounds-timeline payload is writable as a WindowEvent. data = scope.window_event_data() assert data["state"]["viewport"] == [image.width, image.height] + scope.close() @pytest.mark.skipif( not _PRODUCTION_QUALIFICATION, @@ -1896,7 +2237,7 @@ def test_live_window_frame_and_translation(self): ) def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): """Prove live move/resize normalization without changing final app state.""" - discovery_scope = self._scope() + discovery_scope = self._scope(require_on_screen=True) target = discovery_scope.resolve() assert target.title.strip(), ( "production qualification requires a target with a stable window title" @@ -1960,6 +2301,8 @@ def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): assert restored_changed is True assert restored_image.size == tuple(initial_viewport) assert restored_data["window_id"] == str(target.window_id) + scope.close() + discovery_scope.close() def test_live_missing_window_fails_loud(self): scope = WindowCaptureScope(WindowTarget(owner="no-such-app-obviously-not-running-xyz")) From 7b46be0d2a6cfee7cc107bdeb5a352f48dbd8a46 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 30 Aug 2026 21:18:55 -0400 Subject: [PATCH 3/5] fix(macos): fail closed on stale window capture --- README.md | 4 +- docs/WINDOW_CAPTURE.md | 5 +- openadapt_capture/recorder.py | 235 ++++++------ openadapt_capture/window_capture.py | 229 +++++++++-- tests/test_window_capture.py | 566 ++++++++++++++++++++++++++++ 5 files changed, 898 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index 142c8b9..bd244be 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,9 @@ another Space. It does not need to be frontmost or reported as on screen. The recorder accepts complete frames and proven idle frames. A failed provider is disabled for the rest of that recording, so each frame does not repeat a slow failure. The recorder still needs a logged-in desktop session. A minimized -window must return a valid exact frame. +window is refused because macOS can expose stale backing pixels for it. For an +off-Space window, macOS Accessibility must confirm that the exact window is not +minimized. Linux window mode needs X11 with EWMH and XComposite. It refuses to start under native Wayland or XWayland-only. diff --git a/docs/WINDOW_CAPTURE.md b/docs/WINDOW_CAPTURE.md index 6c740cf..166e0d3 100644 --- a/docs/WINDOW_CAPTURE.md +++ b/docs/WINDOW_CAPTURE.md @@ -52,8 +52,9 @@ In this mode: image API and `/usr/sbin/screencapture -o -l` remain exact-window compatibility paths. A failed provider stays disabled for that recording. This supports an occluded window and a window on another Space. It also - supports a window that macOS does not report as on screen. A minimized window - must still return a valid exact frame or the session fails. + supports a window that macOS does not report as on screen. macOS Accessibility + must confirm that an off-screen exact window is not minimized. The recorder + refuses a minimized window because its backing pixels can be stale. Linux X11 reads an XComposite named-window pixmap. It doesn't use a root screenshot, so another window cannot replace the target pixels. Windows grabs the window's screen region, so keep the window unoccluded. diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 32195bc..7876c26 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -2696,123 +2696,132 @@ def record( window_title or config.RECORD_WINDOW_TITLE, frame_rate=config.SCREEN_CAPTURE_FPS, ) - initial_window_frame = None - display_scope = DesktopCaptureScope.current() - desktop_scope = None - if window_scope is not None: - window_scope.bind_display_topology( - display_scope.snapshot(), - display_scope.assert_current, - ) - initial_window_frame, _ = window_scope.capture_frame(publish=False) - window_snapshot = window_scope.snapshot() - logger.info( - "window-scoped capture resolved: window_id={} provider={} " - "visibility_independent={} viewport={}", - window_snapshot.get("window_id"), - window_snapshot.get("capture_source"), - window_snapshot.get("visibility_independent"), - initial_window_frame.size, - ) - else: - # MSS monitor zero is the exact combined frame read by - # ``utils.take_screenshot``. Retain its origin and translate native - # input into that same pixel space so secondary monitors with negative - # global coordinates remain aligned with the video. - desktop_scope = display_scope - logger.info(f"virtual desktop capture resolved: {desktop_scope.snapshot()}") - - if structural_observer is None: - structural_observer = create_structural_observer( - enabled=config.RECORD_STRUCTURAL_OBSERVATIONS, - ) + # The caller keeps this dict so it can reap anything this recording leaves + # behind, even when record() itself raises. + task_by_name = {} if child_registry is None else child_registry + writer_queues = [] + try: + initial_window_frame = None + display_scope = DesktopCaptureScope.current() + desktop_scope = None + if window_scope is not None: + window_scope.bind_display_topology( + display_scope.snapshot(), + display_scope.assert_current, + ) + initial_window_frame, _ = window_scope.capture_frame(publish=False) + window_snapshot = window_scope.snapshot() + logger.info( + "window-scoped capture resolved: window_id={} provider={} " + "visibility_independent={} viewport={}", + window_snapshot.get("window_id"), + window_snapshot.get("capture_source"), + window_snapshot.get("visibility_independent"), + initial_window_frame.size, + ) + else: + # MSS monitor zero is the exact combined frame read by + # ``utils.take_screenshot``. Retain its origin and translate native + # input into that same pixel space so secondary monitors with negative + # global coordinates remain aligned with the video. + desktop_scope = display_scope + logger.info(f"virtual desktop capture resolved: {desktop_scope.snapshot()}") + + if structural_observer is None: + structural_observer = create_structural_observer( + enabled=config.RECORD_STRUCTURAL_OBSERVATIONS, + ) - if capture_dir is None: - capture_dir = os.path.join(os.getcwd(), "capture") - recording, db_path = create_recording( - task_description, - capture_dir, - window_capture_info=(window_scope.snapshot() if window_scope is not None else None), - desktop_capture_info=(desktop_scope.snapshot() if desktop_scope is not None else None), - ) - recording_timestamp = recording.timestamp - - # create_recording() established the one shared clock epoch for this - # capture. Every thread producer inherits that epoch. A thread must not - # call set_start_time() again because doing so can place a later frame - # before the retained initial frame in capture time. - - event_q = OrderedEventJournal() - producers_finished = threading.Event() - input_finished = threading.Event() - terminal_frame_finished = threading.Event() - terminal_frame_cancelled = threading.Event() - input_frame_boundary = NativeInputFrameBoundary() - processing_aborted = threading.Event() - if window_scope is not None: - # The preflight frame sizes the fixed stream. Capture again after the - # recording clock starts, then publish pixels and geometry atomically - # before any input observer can bind an action to the epoch. - initial_window_frame, _ = window_scope.capture_frame(publish=False) - initial_generation = window_scope.current_generation() - initial_timestamp = utils.get_timestamp() - event_q.commit_window_frame( - Event( - initial_timestamp, - "screen", - WindowScopedFrame( - image=initial_window_frame, - window_event_data=window_scope.window_event_data(), - geometry_generation=initial_generation, - ), + if capture_dir is None: + capture_dir = os.path.join(os.getcwd(), "capture") + recording, db_path = create_recording( + task_description, + capture_dir, + window_capture_info=( + window_scope.snapshot() if window_scope is not None else None + ), + desktop_capture_info=( + desktop_scope.snapshot() if desktop_scope is not None else None ), - window_scope, - initial_generation, ) - else: - assert desktop_scope is not None - # Publish one clean before-frame before the native observer can accept - # input. The screen thread attaches to the observer boundary for every - # later frame, but it cannot safely win that startup race by itself. - desktop_scope.assert_current(force=True) - initial_desktop_frame = utils.take_screenshot() - desktop_scope.assert_current(force=True) - if initial_desktop_frame is None: - raise WindowCaptureError("the initial desktop screenshot was empty") - event_q.put( - Event( - utils.get_timestamp(), - "screen", - initial_desktop_frame, + recording_timestamp = recording.timestamp + + # create_recording() established the one shared clock epoch for this + # capture. Every thread producer inherits that epoch. A thread must not + # call set_start_time() again because doing so can place a later frame + # before the retained initial frame in capture time. + + event_q = OrderedEventJournal() + producers_finished = threading.Event() + input_finished = threading.Event() + terminal_frame_finished = threading.Event() + terminal_frame_cancelled = threading.Event() + input_frame_boundary = NativeInputFrameBoundary() + processing_aborted = threading.Event() + if window_scope is not None: + # The preflight frame sizes the fixed stream. Capture again after the + # recording clock starts, then publish pixels and geometry atomically + # before any input observer can bind an action to the epoch. + initial_window_frame, _ = window_scope.capture_frame(publish=False) + initial_generation = window_scope.current_generation() + initial_timestamp = utils.get_timestamp() + event_q.commit_window_frame( + Event( + initial_timestamp, + "screen", + WindowScopedFrame( + image=initial_window_frame, + window_event_data=window_scope.window_event_data(), + geometry_generation=initial_generation, + ), + ), + window_scope, + initial_generation, ) - ) - screen_write_q = sq.SynchronizedQueue() - action_write_q = sq.SynchronizedQueue() - window_write_q = sq.SynchronizedQueue() - browser_write_q = sq.SynchronizedQueue() - video_write_q = sq.SynchronizedQueue() - terminate_writers = multiprocessing.Event() - # TODO: save write times to DB; display performance plot in visualize.py - perf_q = sq.SynchronizedQueue() - if terminate_processing is None: - terminate_processing = multiprocessing.Event() - writer_queues = [ - screen_write_q, - action_write_q, - window_write_q, - browser_write_q, - video_write_q, - perf_q, - ] - # The caller keeps this dict so it can reap anything this recording leaves - # behind, even when record() itself raises. - task_by_name = {} if child_registry is None else child_registry - task_started_events = {} - task_errors: queue.Queue = queue.Queue() - # Writes are synchronous, so a child's traceback reaches this queue before - # the child exits, and reading it is never a race against the child. - process_errors = multiprocessing.SimpleQueue() - _screen_timing = _ScreenTimingStats() # running stats, no unbounded list + else: + assert desktop_scope is not None + # Publish one clean before-frame before the native observer can accept + # input. The screen thread attaches to the observer boundary for every + # later frame, but it cannot safely win that startup race by itself. + desktop_scope.assert_current(force=True) + initial_desktop_frame = utils.take_screenshot() + desktop_scope.assert_current(force=True) + if initial_desktop_frame is None: + raise WindowCaptureError("the initial desktop screenshot was empty") + event_q.put( + Event( + utils.get_timestamp(), + "screen", + initial_desktop_frame, + ) + ) + screen_write_q = sq.SynchronizedQueue() + writer_queues.append(screen_write_q) + action_write_q = sq.SynchronizedQueue() + writer_queues.append(action_write_q) + window_write_q = sq.SynchronizedQueue() + writer_queues.append(window_write_q) + browser_write_q = sq.SynchronizedQueue() + writer_queues.append(browser_write_q) + video_write_q = sq.SynchronizedQueue() + writer_queues.append(video_write_q) + terminate_writers = multiprocessing.Event() + # TODO: save write times to DB; display performance plot in visualize.py + perf_q = sq.SynchronizedQueue() + writer_queues.append(perf_q) + if terminate_processing is None: + terminate_processing = multiprocessing.Event() + task_started_events = {} + task_errors: queue.Queue = queue.Queue() + # Writes are synchronous, so a child's traceback reaches this queue before + # the child exits, and reading it is never a race against the child. + process_errors = multiprocessing.SimpleQueue() + _screen_timing = _ScreenTimingStats() # running stats, no unbounded list + except BaseException: + if window_scope is not None: + window_scope.close() + _release_queues(writer_queues) + raise # Nothing this recording starts may outlive it. A surviving child keeps # the standard output it inherited open, and multiprocessing joins live diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index e9ad030..4302109 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -457,6 +457,11 @@ def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: "the target moved or resized while a frame was captured; " "no action can bind to mixed frame geometry" ) + if pre.on_screen and not post.on_screen: + raise WindowCaptureError( + "the target stopped being visible while a frame was captured; " + "the frame's minimized state cannot be proven" + ) win = post if source_image.width <= 0 or source_image.height <= 0: raise WindowCaptureError("window capture returned an empty frame") @@ -924,6 +929,8 @@ def _process_start_time(pid: int) -> float: _MACOS_CAPTURE_TIMEOUT_SECONDS = 15.0 _MACOS_PROVIDER_CHAIN_TIMEOUT_SECONDS = 20.0 _MACOS_SCK_ATTEMPT_TIMEOUT_SECONDS = 12.0 +_MACOS_AX_STATE_TIMEOUT_SECONDS = 1.0 +_MACOS_AX_GEOMETRY_TOLERANCE_POINTS = 2.0 _MACOS_RUNTIME_LOCK = threading.Lock() _MACOS_RUNTIME_READY = False _MACOS_SC_OUTPUT_CLASS: type | None = None @@ -1410,6 +1417,8 @@ def _start(self, window: TargetWindow, *, deadline: float) -> None: + (f" (error {code})" if code is not None else "") ) with self._condition: + if self._closed: + raise WindowCaptureError("ScreenCaptureKit stream is closed") self._stream = stream self._delegate = delegate self._queue = queue @@ -1479,26 +1488,35 @@ def capture( "ScreenCaptureKit timed out waiting for an exact-window frame" ) self._condition.wait(remaining) - if self._error is not None: - raise self._error - self._delivered_sequence = self._sequence - image = self._last_complete_image.copy() - image.info["openadapt_capture_source"] = ( - "macos-screencapturekit-stream" - ) - image.info["openadapt_frame_status"] = ( - "complete" - if self._last_status - == int(ScreenCaptureKit.SCFrameStatusComplete) - else "idle" - ) - image.info["openadapt_frame_display_time"] = self._last_display_time - image.info["openadapt_pixel_display_time"] = ( - self._last_complete_display_time - ) - image.info["openadapt_stream_sequence"] = self._sequence - image.info["openadapt_stream_generation"] = self._generation - return image + captured_error = self._error + if captured_error is None: + self._delivered_sequence = self._sequence + image = self._last_complete_image.copy() + image.info["openadapt_capture_source"] = ( + "macos-screencapturekit-stream" + ) + image.info["openadapt_frame_status"] = ( + "complete" + if self._last_status + == int(ScreenCaptureKit.SCFrameStatusComplete) + else "idle" + ) + image.info["openadapt_frame_display_time"] = ( + self._last_display_time + ) + image.info["openadapt_pixel_display_time"] = ( + self._last_complete_display_time + ) + image.info["openadapt_stream_sequence"] = self._sequence + image.info["openadapt_stream_generation"] = self._generation + if captured_error is not None: + if self._closed: + self._stop_stream( + suppress_errors=True, + deadline=time.monotonic() + _MACOS_CAPTURE_TIMEOUT_SECONDS, + ) + raise captured_error + return image def _stop_stream( self, @@ -1541,15 +1559,25 @@ def _stop_stream( def close(self, *, timeout_seconds: float = _MACOS_CAPTURE_TIMEOUT_SECONDS) -> None: """Stop the stream and wake any waiting capture.""" + deadline = time.monotonic() + max(0.001, timeout_seconds) with self._condition: self._closed = True self._error = WindowCaptureError("ScreenCaptureKit stream was closed") self._condition.notify_all() - with self._capture_lock: + remaining = max(0.0, deadline - time.monotonic()) + if not self._capture_lock.acquire(timeout=remaining): + logger.warning( + "ScreenCaptureKit close timed out waiting for active frame capture; " + "the capture owner will stop the stream" + ) + return + try: self._stop_stream( suppress_errors=True, - deadline=time.monotonic() + max(0.001, timeout_seconds), + deadline=deadline, ) + finally: + self._capture_lock.release() def _capture_window_macos_utility( @@ -1596,6 +1624,113 @@ def _capture_window_macos_utility( return captured +def _macos_window_minimized( + window: TargetWindow, + *, + deadline: float | None = None, +) -> bool | None: + """Return the exact AX window's minimized state, or ``None`` if unknown. + + Quartz reports both minimized and other-Space windows as not on screen. + A minimized window can expose stale backing pixels through an exact-window + API. Off-screen capture therefore requires Accessibility to distinguish a + normal window on another Space from a minimized window. + """ + deadline = deadline or (time.monotonic() + _MACOS_AX_STATE_TIMEOUT_SECONDS) + try: + import ApplicationServices + + set_timeout = ApplicationServices.AXUIElementSetMessagingTimeout + + def attribute(element: object, name: str) -> object | None: + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + timeout_error = set_timeout( + element, + min(_MACOS_AX_STATE_TIMEOUT_SECONDS, remaining), + ) + if timeout_error != ApplicationServices.kAXErrorSuccess: + return None + error, value = ApplicationServices.AXUIElementCopyAttributeValue( + element, + name, + None, + ) + if error != ApplicationServices.kAXErrorSuccess: + return None + return value + + application = ApplicationServices.AXUIElementCreateApplication(window.pid) + candidates = attribute(application, "AXWindows") + if candidates is None: + return None + except Exception: + return None + + id_matches = [] + title_candidates = [] + for candidate in candidates or []: + try: + number = attribute(candidate, "AXWindowNumber") + if number is not None: + if int(number) == window.window_id: + id_matches.append(candidate) + continue + title = attribute(candidate, "AXTitle") + if title is not None and str(title) == window.title: + title_candidates.append(candidate) + except Exception: + continue + + title_matches = [] + if not id_matches: + for candidate in title_candidates: + try: + position_value = attribute(candidate, "AXPosition") + size_value = attribute(candidate, "AXSize") + if position_value is None or size_value is None: + continue + position_ok, position = ApplicationServices.AXValueGetValue( + position_value, + ApplicationServices.kAXValueCGPointType, + None, + ) + size_ok, size = ApplicationServices.AXValueGetValue( + size_value, + ApplicationServices.kAXValueCGSizeType, + None, + ) + if not position_ok or not size_ok: + continue + expected = window.bounds + actual = ( + float(position.x), + float(position.y), + float(size.width), + float(size.height), + ) + if all( + abs(actual[index] - expected[index]) + <= _MACOS_AX_GEOMETRY_TOLERANCE_POINTS + for index in range(4) + ): + title_matches.append(candidate) + except Exception: + continue + + matches = id_matches if id_matches else title_matches + if len(matches) != 1: + return None + try: + minimized = attribute(matches[0], "AXMinimized") + except Exception: + return None + if minimized is None: + return None + return bool(minimized) + + def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: """macOS: CGWindowList by owner/title substring. @@ -1685,6 +1820,8 @@ class _MacOSWindowCaptureProvider: """Session-owned exact-window providers with sticky safe fallback.""" def __init__(self, *, frame_rate: float | None = None) -> None: + self._state_lock = threading.Lock() + self._closed = False self._stream = ( _MacOSScreenCaptureKitStream(frame_rate=frame_rate) if _screen_capture_kit_available() @@ -1693,13 +1830,35 @@ def __init__(self, *, frame_rate: float | None = None) -> None: self._sck_disabled = self._stream is None self._quartz_disabled = False + def _assert_open(self) -> None: + with self._state_lock: + if self._closed: + raise WindowCaptureError("the exact-window capture provider is closed") + def capture(self, window: TargetWindow) -> "Image.Image": """Capture only ``window`` and never substitute desktop pixels.""" failures: list[BaseException] = [] chain_deadline = time.monotonic() + _MACOS_PROVIDER_CHAIN_TIMEOUT_SECONDS + self._assert_open() + + def assert_offscreen_target_is_safe() -> None: + if window.on_screen: + return + minimized = _macos_window_minimized(window, deadline=chain_deadline) + if minimized is True: + raise WindowCaptureUnavailableError( + "the exact target window is minimized; restore it before recording" + ) + if minimized is None: + raise WindowCaptureUnavailableError( + "the exact off-screen target could not be proven non-minimized; " + "grant Accessibility access or move the window to the active Space" + ) + + assert_offscreen_target_is_safe() if not self._sck_disabled and self._stream is not None: try: - return self._stream.capture( + captured = self._stream.capture( window, deadline=min( chain_deadline, @@ -1707,6 +1866,7 @@ def capture(self, window: TargetWindow) -> "Image.Image": ), ) except WindowCaptureError as exc: + self._assert_open() failures.append(exc) self._stream.close(timeout_seconds=2.0) self._sck_disabled = True @@ -1714,8 +1874,13 @@ def capture(self, window: TargetWindow) -> "Image.Image": "ScreenCaptureKit exact-window stream failed; disabling it " "for this recording session" ) + else: + self._assert_open() + assert_offscreen_target_is_safe() + return captured if window.on_screen and not self._quartz_disabled: + self._assert_open() try: import Quartz @@ -1726,6 +1891,7 @@ def capture(self, window: TargetWindow) -> "Image.Image": Quartz.kCGWindowImageBoundsIgnoreFraming, ) except Exception as exc: + self._assert_open() img_ref = None failures.append( WindowCaptureUnavailableError( @@ -1734,11 +1900,14 @@ def capture(self, window: TargetWindow) -> "Image.Image": ) logger.debug("Quartz exact-window capture failed: {}", exc) if img_ref is not None: - return _pil_image_from_cgimage( + captured = _pil_image_from_cgimage( img_ref, source="macos-quartz-window-image", ) + self._assert_open() + return captured self._quartz_disabled = True + self._assert_open() failures.append( WindowCapturePermissionError( f"Quartz returned no image for exact window {window.window_id}" @@ -1756,16 +1925,22 @@ def capture(self, window: TargetWindow) -> "Image.Image": ) try: + self._assert_open() + assert_offscreen_target_is_safe() remaining = chain_deadline - time.monotonic() if remaining <= 0: raise WindowCaptureUnavailableError( "the exact-window provider chain exhausted its startup budget" ) - return _capture_window_macos_utility( + captured = _capture_window_macos_utility( window, timeout_seconds=min(_MACOS_CAPTURE_TIMEOUT_SECONDS, remaining), ) + self._assert_open() + assert_offscreen_target_is_safe() + return captured except WindowCaptureError as exc: + self._assert_open() failures.append(exc) failure_type = ( WindowCapturePermissionError @@ -1790,6 +1965,10 @@ def capture(self, window: TargetWindow) -> "Image.Image": def close(self) -> None: """Stop the owned stream, if it started.""" + with self._state_lock: + if self._closed: + return + self._closed = True if self._stream is not None: self._stream.close() diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index cca7b6b..1267056 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -52,6 +52,65 @@ translate_point, ) + +def test_record_closes_window_scope_when_startup_fails(monkeypatch, tmp_path): + """A preflight stream must not survive a later recorder setup failure.""" + + class FakeWindowScope: + def __init__(self): + self.close_calls = 0 + + def bind_display_topology(self, _snapshot, _guard): + return None + + def capture_frame(self, *, publish=False): + assert publish is False + return Image.new("RGB", (100, 80), "white"), True + + def snapshot(self): + return { + "window_id": 42, + "capture_source": "test-exact-window-stream", + "visibility_independent": True, + } + + def close(self): + self.close_calls += 1 + + fake_scope = FakeWindowScope() + fake_display_scope = SimpleNamespace( + snapshot=lambda: {"topology_sha256": "test-topology"}, + assert_current=lambda **_kwargs: None, + ) + monkeypatch.setattr(recorder_module.config, "RECORD_BROWSER_EVENTS", False) + monkeypatch.setattr(recorder_module.config, "RECORD_VIDEO", False) + monkeypatch.setattr(recorder_module.config, "RECORD_IMAGES", True) + monkeypatch.setattr( + recorder_module, + "build_window_scope", + lambda *_args, **_kwargs: fake_scope, + ) + monkeypatch.setattr( + recorder_module.DesktopCaptureScope, + "current", + lambda: fake_display_scope, + ) + monkeypatch.setattr( + recorder_module, + "create_recording", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("setup failed")), + ) + + with pytest.raises(RuntimeError, match="setup failed"): + recorder_module.record( + "startup failure cleanup", + capture_dir=str(tmp_path), + structural_observer=object(), + ) + + assert fake_scope.close_calls == 1 + + # --------------------------------------------------------------------------- # translate_point: exact inverse of flow's replay mapping # --------------------------------------------------------------------------- @@ -1106,6 +1165,11 @@ def close(self, **_kwargs): stream_calls.append("close") monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: False, + ) monkeypatch.setattr( window_capture_module, "_MacOSScreenCaptureKitStream", @@ -1152,6 +1216,11 @@ def close(self, **_kwargs): stream_calls.append("close") monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: False, + ) monkeypatch.setattr( window_capture_module, "_MacOSScreenCaptureKitStream", @@ -1366,6 +1435,83 @@ def wait_for_frame(): assert "closed" in str(errors[0]) +def test_screencapturekit_close_budget_includes_capture_lock_wait(): + stream = window_capture_module._MacOSScreenCaptureKitStream() + lock_held = threading.Event() + release_lock = threading.Event() + + def hold_capture_lock(): + with stream._capture_lock: + lock_held.set() + assert release_lock.wait(timeout=1) + + holder = threading.Thread(target=hold_capture_lock) + holder.start() + assert lock_held.wait(timeout=1) + started = time.monotonic() + stream.close(timeout_seconds=0.02) + elapsed = time.monotonic() - started + release_lock.set() + holder.join(timeout=1) + + assert elapsed < 0.2 + assert stream._closed is True + assert not holder.is_alive() + + +def test_screencapturekit_capture_owner_stops_stream_after_close_timeout(monkeypatch): + _fake_screencapturekit_modules(monkeypatch, {}) + stop_called = threading.Event() + allow_stop = threading.Event() + stop_calls = 0 + + class FakeNativeStream: + @staticmethod + def stopCaptureWithCompletionHandler_(callback): + nonlocal stop_calls + stop_calls += 1 + stop_called.set() + assert allow_stop.wait(timeout=1) + callback(None) + + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 20.0, 10.0), + process_start_time=123.0, + ) + stream = window_capture_module._MacOSScreenCaptureKitStream() + stream._stream = FakeNativeStream() + stream._window_id = window.window_id + stream._bounds_size = (window.bounds[2], window.bounds[3]) + stream._generation = 1 + errors = [] + + def wait_for_frame(): + try: + stream.capture(window, deadline=time.monotonic() + 5) + except WindowCaptureError as exc: + errors.append(exc) + + capture_thread = threading.Thread(target=wait_for_frame) + capture_thread.start() + time.sleep(0.02) + started = time.monotonic() + stream.close(timeout_seconds=0.02) + elapsed = time.monotonic() - started + assert stop_called.wait(timeout=1) + allow_stop.set() + capture_thread.join(timeout=1) + + assert elapsed < 0.2 + assert not capture_thread.is_alive() + assert len(errors) == 1 + assert stream._stream is None + assert stop_calls == 1 + + def test_macos_provider_preserves_all_provider_permission_denial(monkeypatch): window = TargetWindow( window_id=19373, @@ -1390,6 +1536,11 @@ def close(self, **_kwargs): pass monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: False, + ) monkeypatch.setattr( window_capture_module, "_MacOSScreenCaptureKitStream", @@ -1407,6 +1558,393 @@ def close(self, **_kwargs): window_capture_module._MacOSWindowCaptureProvider().capture(window) +def test_macos_provider_refuses_minimized_window_before_capture(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: True, + ) + utility_calls = [] + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: utility_calls.append("capture"), + ) + + with pytest.raises(WindowCaptureError, match="minimized"): + window_capture_module._MacOSWindowCaptureProvider().capture(window) + + assert utility_calls == [] + + +def test_macos_provider_refuses_unproven_offscreen_capture(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: None, + ) + utility_calls = [] + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: utility_calls.append("capture"), + ) + + with pytest.raises(WindowCaptureError, match="proven non-minimized"): + window_capture_module._MacOSWindowCaptureProvider().capture(window) + + assert utility_calls == [] + + +def test_macos_provider_refuses_capture_after_close(monkeypatch): + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + provider = window_capture_module._MacOSWindowCaptureProvider() + provider.close() + utility_calls = [] + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: utility_calls.append("capture"), + ) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=True, + process_start_time=123.0, + ) + + with pytest.raises(WindowCaptureError, match="provider is closed"): + provider.capture(window) + + assert utility_calls == [] + + +def test_macos_provider_close_during_stream_capture_cannot_fallback(monkeypatch): + capture_started = threading.Event() + release_capture = threading.Event() + utility_calls = [] + + class BlockingStream: + def __init__(self, **_kwargs): + pass + + def capture(self, _window, **_kwargs): + capture_started.set() + assert release_capture.wait(timeout=1) + return Image.new("RGB", (20, 10), "white") + + def close(self, **_kwargs): + release_capture.set() + + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: True) + monkeypatch.setattr( + window_capture_module, + "_MacOSScreenCaptureKitStream", + BlockingStream, + ) + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: utility_calls.append("capture"), + ) + provider = window_capture_module._MacOSWindowCaptureProvider() + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 20.0, 10.0), + on_screen=True, + process_start_time=123.0, + ) + errors = [] + + def capture(): + try: + provider.capture(window) + except WindowCaptureError as exc: + errors.append(exc) + + capture_thread = threading.Thread(target=capture) + capture_thread.start() + assert capture_started.wait(timeout=1) + provider.close() + capture_thread.join(timeout=1) + + assert not capture_thread.is_alive() + assert len(errors) == 1 + assert "provider is closed" in str(errors[0]) + assert utility_calls == [] + + +def test_macos_provider_rechecks_minimized_state_before_utility(monkeypatch): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + states = iter((False, True)) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: next(states), + ) + utility_calls = [] + monkeypatch.setattr( + window_capture_module, + "_capture_window_macos_utility", + lambda *_args, **_kwargs: utility_calls.append("capture"), + ) + + with pytest.raises(WindowCaptureError, match="all exact-window") as exc_info: + window_capture_module._MacOSWindowCaptureProvider().capture(window) + + assert utility_calls == [] + assert any("minimized" in note for note in exc_info.value.__notes__) + + +def test_macos_provider_refuses_utility_frame_if_window_becomes_minimized( + monkeypatch, +): + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + coordinate_source="quartz-screen-points", + visibility_independent=True, + ) + states = iter((False, False, True)) + monkeypatch.setattr(window_capture_module, "_screen_capture_kit_available", lambda: False) + monkeypatch.setattr( + window_capture_module, + "_macos_window_minimized", + lambda _window, **_kwargs: next(states), + ) + utility_calls = [] + + def utility(*_args, **_kwargs): + utility_calls.append("capture") + return Image.new("RGB", (3024, 1888), "white") + + monkeypatch.setattr(window_capture_module, "_capture_window_macos_utility", utility) + + with pytest.raises(WindowCaptureError, match="all exact-window") as exc_info: + window_capture_module._MacOSWindowCaptureProvider().capture(window) + + assert utility_calls == ["capture"] + assert any("minimized" in note for note in exc_info.value.__notes__) + + +def test_macos_minimized_state_matches_exact_ax_window_number(monkeypatch): + exact = object() + other = object() + attributes = { + (exact, "AXWindowNumber"): 19373, + (exact, "AXTitle"): "Document", + (exact, "AXMinimized"): False, + (other, "AXWindowNumber"): 99, + (other, "AXTitle"): "Document", + } + + def copy_attribute(element, name, _value): + if element == "application" and name == "AXWindows": + return 0, [other, exact] + value = attributes.get((element, name)) + return (0, value) if value is not None else (1, None) + + application_services = SimpleNamespace( + kAXErrorSuccess=0, + AXUIElementCreateApplication=lambda _pid: "application", + AXUIElementSetMessagingTimeout=lambda _element, _timeout: 0, + AXUIElementCopyAttributeValue=copy_attribute, + ) + monkeypatch.setitem(sys.modules, "ApplicationServices", application_services) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + ) + + assert window_capture_module._macos_window_minimized(window) is False + + +def test_macos_minimized_title_fallback_requires_matching_geometry(monkeypatch): + candidate = object() + attributes = { + (candidate, "AXTitle"): "Document", + (candidate, "AXPosition"): SimpleNamespace(x=50.0, y=50.0), + (candidate, "AXSize"): SimpleNamespace(width=1512.0, height=944.0), + (candidate, "AXMinimized"): False, + } + + def copy_attribute(element, name, _value): + if element == "application" and name == "AXWindows": + return 0, [candidate] + value = attributes.get((element, name)) + return (0, value) if value is not None else (1, None) + + application_services = SimpleNamespace( + kAXErrorSuccess=0, + kAXValueCGPointType=1, + kAXValueCGSizeType=2, + AXUIElementCreateApplication=lambda _pid: "application", + AXUIElementSetMessagingTimeout=lambda _element, _timeout: 0, + AXUIElementCopyAttributeValue=copy_attribute, + AXValueGetValue=lambda value, _type, _output: (True, value), + ) + monkeypatch.setitem(sys.modules, "ApplicationServices", application_services) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + ) + + assert window_capture_module._macos_window_minimized(window) is None + + +def test_macos_minimized_title_fallback_rejects_known_different_window_id( + monkeypatch, +): + candidate = object() + attributes = { + (candidate, "AXWindowNumber"): 99, + (candidate, "AXTitle"): "Document", + (candidate, "AXPosition"): SimpleNamespace(x=0.0, y=0.0), + (candidate, "AXSize"): SimpleNamespace(width=1512.0, height=944.0), + (candidate, "AXMinimized"): False, + } + + def copy_attribute(element, name, _value): + if element == "application" and name == "AXWindows": + return 0, [candidate] + value = attributes.get((element, name)) + return (0, value) if value is not None else (1, None) + + application_services = SimpleNamespace( + kAXErrorSuccess=0, + kAXValueCGPointType=1, + kAXValueCGSizeType=2, + AXUIElementCreateApplication=lambda _pid: "application", + AXUIElementSetMessagingTimeout=lambda _element, _timeout: 0, + AXUIElementCopyAttributeValue=copy_attribute, + AXValueGetValue=lambda value, _type, _output: (True, value), + ) + monkeypatch.setitem(sys.modules, "ApplicationServices", application_services) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + ) + + assert window_capture_module._macos_window_minimized(window) is None + + +def test_macos_minimized_lookup_sets_bounded_ax_timeout(monkeypatch): + timeouts = [] + + def set_timeout(_element, timeout): + timeouts.append(timeout) + return 0 + + application_services = SimpleNamespace( + kAXErrorSuccess=0, + AXUIElementCreateApplication=lambda _pid: "application", + AXUIElementSetMessagingTimeout=set_timeout, + AXUIElementCopyAttributeValue=lambda *_args: (1, None), + ) + monkeypatch.setitem(sys.modules, "ApplicationServices", application_services) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + ) + deadline = time.monotonic() + 0.05 + + assert window_capture_module._macos_window_minimized( + window, + deadline=deadline, + ) is None + assert len(timeouts) == 1 + assert 0 < timeouts[0] <= 0.05 + + +def test_macos_minimized_lookup_refuses_rejected_ax_timeout(monkeypatch): + reads = [] + application_services = SimpleNamespace( + kAXErrorSuccess=0, + AXUIElementCreateApplication=lambda _pid: "application", + AXUIElementSetMessagingTimeout=lambda _element, _timeout: 1, + AXUIElementCopyAttributeValue=lambda *_args: reads.append("read"), + ) + monkeypatch.setitem(sys.modules, "ApplicationServices", application_services) + window = TargetWindow( + window_id=19373, + owner="FakeApp", + title="Document", + pid=100, + bounds=(0.0, 0.0, 1512.0, 944.0), + on_screen=False, + process_start_time=123.0, + ) + + assert window_capture_module._macos_window_minimized(window) is None + assert reads == [] + + class TestTranslatePoint: """Coordinate translation: global screen points -> window pixels.""" @@ -1504,6 +2042,8 @@ def __init__(self, bounds=(300.0, 150.0, 800.0, 600.0), scale=2.0): self.window_id = 42 self.title = "Fake Window" self.missing = False + self.on_screen = True + self.visibility_independent = False self.process_start_time = 123.5 def resolver(self, target: WindowTarget): @@ -1515,6 +2055,8 @@ def resolver(self, target: WindowTarget): title=self.title, pid=1234, bounds=self.bounds, + on_screen=self.on_screen, + visibility_independent=self.visibility_independent, process_start_time=self.process_start_time, coordinate_source="test-screen-points", ) @@ -1555,6 +2097,30 @@ def test_capture_frame_returns_window_pixels(self, scope): assert changed is True # first frame always establishes the timeline assert image.size == (1600, 1200) # 800x600 points at 2x + def test_visible_window_cannot_become_offscreen_during_capture(self, fake): + fake.visibility_independent = True + + def capturer(window): + image = fake.capturer(window) + fake.on_screen = False + return image + + capture_scope = WindowCaptureScope( + WindowTarget(owner="FakeApp"), + resolver=fake.resolver, + capturer=capturer, + ) + capture_scope.bind_display_topology( + { + "schema_version": "openadapt.capture.display-topology/v1", + "topology_sha256": "a" * 64, + }, + lambda **_kwargs: None, + ) + + with pytest.raises(WindowCaptureError, match="stopped being visible"): + capture_scope.capture_frame() + def test_scale_computed_from_frame_and_bounds(self, scope): scope.capture_frame() assert scope.snapshot()["scale"] == 2.0 From 73202cdba3d7938bcad545cf2fd1be73e221471b Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 31 Aug 2026 00:59:01 -0400 Subject: [PATCH 4/5] feat: add protected authentication handoffs --- CHANGELOG.md | 9 + README.md | 31 +- docs/AUTHENTICATION_HANDOFF.md | 184 +++++++ docs/DESIGN.md | 54 ++- openadapt_capture/__init__.py | 22 + openadapt_capture/authentication.py | 694 +++++++++++++++++++++++++++ openadapt_capture/capture.py | 127 +++++ openadapt_capture/control.py | 132 ++++- openadapt_capture/recorder.py | 557 ++++++++++++++++----- tests/test_authentication_handoff.py | 499 +++++++++++++++++++ tests/test_control.py | 105 ++++ 11 files changed, 2275 insertions(+), 139 deletions(-) create mode 100644 docs/AUTHENTICATION_HANDOFF.md create mode 100644 openadapt_capture/authentication.py create mode 100644 tests/test_authentication_handoff.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83cff7c..8ee909f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ +## Unreleased + +### Features + +- Add attended authentication handoffs. Capture suppresses sensitive source + data, seals a bounded method marker, and retains a fresh exact frame before + native input resumes. The same retry-safe operations are available through + the authenticated local control channel. + ## v1.3.0 (2026-08-28) _This release is published under the MIT License._ diff --git a/README.md b/README.md index bd244be..7531190 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ recorder as a library. [Documentation](https://docs.openadapt.ai) · [Flow](https://github.com/OpenAdaptAI/openadapt-flow) · -[Window capture](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/WINDOW_CAPTURE.md) +[Window capture](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/WINDOW_CAPTURE.md) · +[Authentication handoffs](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/AUTHENTICATION_HANDOFF.md) ## With OpenAdapt @@ -88,6 +89,7 @@ A finished capture looks like this: ```text my-capture/ +├── authentication-handoffs.json ├── capture-state.json ├── recording.db ├── oa_recording-*.mp4 @@ -108,6 +110,30 @@ an authenticated session capability that lives in an owner-only runtime file On macOS it also removes extended ACL entries and verifies they are absent. The capability never reaches command arguments, logs, or the capture directory. +## Authentication inside a recording + +Authenticate before recording when the demonstration does not need the login. +When login belongs inside the workflow, start an authentication handoff. Capture +then drops screen, input, structural, window, browser-category, and microphone +content at the source boundary. It stores a sealed marker with method classes +and booleans. It has no field for a credential or account identifier. + +```python +handoff = recorder.begin_authentication( + methods="password_manager", + requires_user_presence=True, + saved_account_selected=True, +) +# Select the saved account and finish any MFA prompt. +marker = recorder.end_authentication(handoff, outcome="completed") +``` + +The end call returns after Capture retains a new exact frame. Input stays +blocked if that frame fails. A launcher can use the authenticated +`begin_authentication_handoff()` and `end_authentication_handoff()` process +control functions for the same contract. See +[the authentication handoff contract](https://github.com/OpenAdaptAI/openadapt-capture/blob/main/docs/AUTHENTICATION_HANDOFF.md). + ## FFmpeg Recording video needs an FFmpeg executable, and `capture install-ffmpeg` gets @@ -232,6 +258,9 @@ A raw capture can hold everything visible on screen and everything typed: credentials, personal data, protected health information. Keep the whole directory inside its approved local boundary and give it a retention policy. +An authentication handoff prevents the named login interval from entering the +raw capture. Content retained before or after that interval remains sensitive. + Capture doesn't upload a session. The sharing command and the profiling transfer are the only two transfers and both are explicit. Installing the `privacy` extra does not scrub anything by itself. diff --git a/docs/AUTHENTICATION_HANDOFF.md b/docs/AUTHENTICATION_HANDOFF.md new file mode 100644 index 0000000..544a5c1 --- /dev/null +++ b/docs/AUTHENTICATION_HANDOFF.md @@ -0,0 +1,184 @@ +# Authentication handoffs + +An authentication handoff stops sensitive source retention while a person, +password manager, passkey provider, SSO page, or MFA device completes a login. +Capture keeps the recording open. It stores a small timeline marker and resumes +normal recording only after it has retained a new exact frame. + +Authenticate before recording when that produces a complete demonstration. +Use a handoff when authentication must occur inside the workflow or when a +session can expire during a long recording. + +## What Capture suppresses + +The protected interval covers every source owned by the native recorder: + +| Source | Behavior during the handoff | +| --- | --- | +| Screen and video | Capture retains no frame. The encoded video holds its prior visual state until the fresh resume frame. | +| Mouse and keyboard | The native observer drops the event before journal reservation and before any structural lookup. | +| Accessibility | UIA, AX, and AT-SPI observations do not run for protected input. | +| Window metadata | Capture does not retain active-window titles, bounds, or state. | +| Audio | The audio process replaces microphone chunks with generated silence so the audio clock stays aligned. It does not retain the protected waveform. | +| Browser category | The marker declares browser data suppressed. The supported Playwright browser recorder remains in `openadapt-flow` and needs its own matching source boundary. | + +The recorder does not add black frames or synthetic action rows. A black frame +can look like a real application state. An action row without pixels can look +replayable. The sidecar names the gap directly. + +## What the marker contains + +`authentication-handoffs.json` uses +`openadapt.capture.authentication-handoffs/v1`. The capture seal inventories +its exact bytes. Each interval contains: + +- a random interval UUID; +- one or more method classes from `password_manager`, `passkey`, `sso`, `mfa`, + `device_unlock`, and `other`; +- `requires_user_presence` and `saved_account_selected` booleans; +- start and end times on the capture clock; +- an outcome: `completed`, `cancelled`, `failed`, or `aborted`; +- the fixed list of suppressed source categories; and +- the timestamp, source ordinal, pixel digest, capture source, and optional + window geometry generation of the clean entry frame; +- for a normal close, the same proof fields for the fresh resume frame. + +The API has no field for a provider name, account identifier, username, +password, passkey material, OTP, recovery code, vault item, or free-form note. +Method classes describe the handoff without describing the credential. + +An autofill login fits this contract. Set `methods="password_manager"` and +`saved_account_selected=True`. The click on the saved account, the account +chooser, and any filled values stay inside the protected interval. + +## Recorder API + +The owner can control a recorder in the same process: + +```python +from openadapt_capture import Recorder + +with Recorder( + "./capture", + task_description="Download the monthly statement", + window={"owner": "Google Chrome", "title": "American Express"}, +) as recorder: + if not recorder.wait_for_ready(): + raise RuntimeError("Capture did not become ready") + + handoff = recorder.begin_authentication( + methods="password_manager", + requires_user_presence=True, + saved_account_selected=True, + ) + + # The person selects the saved account and completes any OS or MFA prompt. + # Wait until the application no longer shows credential UI before ending. + + marker = recorder.end_authentication( + handoff, + outcome="completed", + timeout=10, + ) + assert marker.resume_frame is not None +``` + +`begin_authentication()` returns only after in-flight screen, input, window, +and structural operations finish. When audio is active, it also waits for the +microphone process to acknowledge suppression after its callback lock is clear. +The screen worker retains one clean entry frame before the open marker is +written. This closes the after-frame binding for an action that occurred just +before the handoff. New input stays blocked across that cut. + +If that begin barrier times out or the open marker cannot be written, the +recorder stays protected and fails the recording. Resuming would create an +unmarked evidence gap. + +`end_authentication()` first changes the interval to a resuming state. Input, +window metadata, structural data, and audio remain protected. The screen worker +then acquires and journals one new frame. For window-scoped recording, both the +entry and resume frames use the same source ordinal as their exact geometry. +Capture writes the resume proof to the sidecar before it reopens the other +sources. + +If the timeout expires, the interval stays protected and the caller can repeat +the same end request. A source-frame error follows Capture's fail-loud media +rule and ends the recording. Capture never resumes input because a timer +expired or a frame failed. + +## Authenticated process control + +A launcher, Desktop, or Flow process can use the owner-only local control +channel: + +```python +from openadapt_capture import ( + begin_authentication_handoff, + end_authentication_handoff, +) + +handoff = begin_authentication_handoff( + session_id=session_id, + methods=("password_manager", "mfa"), + requires_user_presence=True, + saved_account_selected=True, +) + +# Complete the attended login, then close the protected interval. +marker = end_authentication_handoff( + handoff, + session_id=session_id, + outcome="completed", +) +``` + +These requests use the same loopback capability, process identity, message MAC, +clock bound, and replay checks as `status_recording()` and `stop_recording()`. +The begin client selects the interval UUID. A retry with the same UUID and the +same parameters returns the same handle. A retry with different parameters is +refused. End is idempotent for the same interval and outcome. + +`RecorderStatus.authentication_protected` lets an owner show a local privacy +indicator. It does not expose method classes or account data. + +## Stop and failure behavior + +Stopping a recorder during a handoff closes the interval as `aborted`. It does +not take a terminal screenshot from the credential UI and does not claim a +resume frame. The rest of the capture can still finish and seal if its retained +evidence is valid. + +Stopping during the entry cut, before the open marker exists, fails the +recording. Capture cannot seal a source gap that has no interval marker. + +Nested handoffs are refused. An end call with the wrong handle is refused. A +method outside the fixed vocabulary is refused before the interval starts. +Current captures cannot contain an open interval at the completion seal. + +`CaptureSession.load_verified()` checks each interval against the immutable +database. Both proofs must name retained screen rows with the same timestamps +and source ordinals. The entry proof must be the last pre-handoff frame. No +action, browser, window, or other screen row can occur inside the protected +range. Window-scoped proof must match the frame's capture source and geometry +generation. When PNG pixels are present, their source-pixel digest must match +the marker. + +## Ownership boundary + +Capture proves observation behavior. It proves that the named sources were not +retained and that a fresh frame existed before normal observation resumed. + +Capture does not prove that the application accepted the login. The +`completed` outcome means that the owner completed the handoff. A compiler or +runtime must verify the application state before it treats the user as +authenticated. That contract belongs outside `openadapt-capture`. + +Capture also does not store credentials or ask a password manager to release a +secret. Password managers, passkey providers, operating-system account choosers, +and MFA devices keep that authority. OpenAdapt can coordinate the attended +gate while those systems retain credential custody. + +The native handoff does not turn the repository-only Chrome extension into a +supported recorder. The Playwright path must suppress DOM values, page frames, +network-derived observations, and browser input at its own source boundary. +The two recorders can share the schema after that browser contract exists. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index a503cb4..ad7b8db 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -42,20 +42,23 @@ One recording has these stages: 2. Resolve the initial native-window or virtual-desktop coordinate scope. 3. Create the per-capture SQLite database and media staging path. 4. Observe native input and screen frames on separate workers. -5. Reserve each observation in one ordered source journal before any optional +5. Apply the optional authentication gate before source retention. An open gate + drops screen, input, structure, window data, browser-category data, and audio + content. A close waits for one fresh frame. +6. Reserve each observation in one ordered source journal before any optional structural lookup. A failed reservation fails the session; later events cannot pass it. -6. For a native window, enqueue each frame and its window geometry as one +7. For a native window, enqueue each frame and its window geometry as one source-ordinal pair. Publish that geometry to input observers only after the pair enters the journal. -7. Bind each actionable input to the last published frame pair and optional +8. Bind each actionable input to the last published frame pair and optional structural observation. Retain one ordinal-later frame after input stops. -8. Stream RGB frames to a separately provisioned FFmpeg process. -9. Close, verify, and atomically promote the MP4. Retain an incomplete partial +9. Stream RGB frames to a separately provisioned FFmpeg process. +10. Close, verify, and atomically promote the MP4. Retain an incomplete partial file on an encoder failure and never report it as complete media. -10. Post-process raw input into the public action view. A merged action keeps +11. Post-process raw input into the public action view. A merged action keeps its terminal primitive's source binding and refuses mixed geometry epochs. -11. Reconcile committed rows with producer counts, verify the v2 frame/action +12. Reconcile committed rows with producer counts, verify the v2 frame/action relations, inventory every immutable artifact, and write the completion seal. @@ -104,7 +107,42 @@ Each current window frame carries a process-bound window identity, display topology digest, and geometry epoch digest. The window and screenshot rows use the same source ordinal. An action uses a later ordinal and names the exact pair that supplied its coordinates. Capture refuses process replacement, topology -drift, off-screen state, mixed generations, or a missing pair. +drift, a minimized or unproven off-screen state, mixed generations, or a +missing pair. A desktop-independent exact-window provider can capture an +occluded window or a window on another macOS Space. + +## Authentication observation boundary + +The default operating procedure authenticates before recording. Some workflows +need a login step, and long sessions can expire. Capture supports those cases +with a source-time protected interval. + +The controller enters protection before it writes the open marker. It waits for +each in-flight source operation to finish. Native input holds that boundary from +receipt-time journal reservation through delivery, so a slow structural query +cannot cross the marker. Audio uses a process-shared suppression event and +inserts generated silence to preserve its clock. The parent waits for an audio +acknowledgement taken under the callback lock before it writes the start time. +Failure to reach this cut stops the recording. It cannot resume with an +unmarked gap. + +Before the marker starts, the screen worker journals one clean entry frame. +This frame closes any pending after-frame relationship from normal input. New +input remains blocked while that frame enters the journal. + +Normal close keeps all sources protected while the screen worker acquires a new +frame. The frame enters the ordered journal before the marker records its +source ordinal and pixel digest. A window-scoped frame also binds its geometry +generation. The controller persists the close marker before it reopens input. + +`authentication-handoffs.json` is canonical JSON and part of the immutable +artifact inventory. The marker uses a fixed method vocabulary and accepts no +free text. A stopped interval becomes `aborted` and has no resume claim. + +Capture proves suppression and frame reacquisition. It does not prove semantic +login success and does not hold credential authority. See +[`AUTHENTICATION_HANDOFF.md`](AUTHENTICATION_HANDOFF.md) for the API, schema, +control-channel behavior, and loader checks. ## Completion and consumer boundary diff --git a/openadapt_capture/__init__.py b/openadapt_capture/__init__.py index 3a8203d..83d5983 100644 --- a/openadapt_capture/__init__.py +++ b/openadapt_capture/__init__.py @@ -15,6 +15,16 @@ # still be inspected. The repository-only Chrome-extension bridge is not part # of the production package or API. Supported browser recording is owned by # openadapt-flow's Playwright launch and attach paths. +from openadapt_capture.authentication import ( + AUTHENTICATION_HANDOFF_FILENAME, + AuthenticationBoundaryError, + AuthenticationHandoff, + AuthenticationHandoffError, + AuthenticationHandoffHandle, + AuthenticationHandoffManifest, + FreshFrameProof, + load_authentication_handoffs, +) from openadapt_capture.browser_events import ( BoundingBox, BrowserClickEvent, @@ -47,7 +57,9 @@ CaptureControlError, CaptureControlUnavailable, RecorderStatus, + begin_authentication_handoff, discover_recorders, + end_authentication_handoff, status_recording, stop_recording, ) @@ -147,15 +159,25 @@ "Recorder", "RecordingConfig", "RecorderStatus", + "begin_authentication_handoff", "CaptureControlError", "CaptureControlUnavailable", "CaptureControlAuthenticationError", "discover_recorders", + "end_authentication_handoff", "status_recording", "stop_recording", "Capture", "CaptureSession", "Action", + "AUTHENTICATION_HANDOFF_FILENAME", + "AuthenticationBoundaryError", + "AuthenticationHandoff", + "AuthenticationHandoffError", + "AuthenticationHandoffHandle", + "AuthenticationHandoffManifest", + "FreshFrameProof", + "load_authentication_handoffs", # Native structural observation "STRUCTURAL_OBSERVATION_SCHEMA_VERSION", "StructuralAncestor", diff --git a/openadapt_capture/authentication.py b/openadapt_capture/authentication.py new file mode 100644 index 0000000..a5b65fc --- /dev/null +++ b/openadapt_capture/authentication.py @@ -0,0 +1,694 @@ +"""Source-time protection for attended authentication handoffs. + +Capture owns the observation boundary. It does not own credentials or decide +whether an application is authenticated. This module suppresses sensitive +sources during an attended handoff and retains a small, sealed timeline marker. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import multiprocessing +import os +import tempfile +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Literal, Sequence + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +AUTHENTICATION_HANDOFF_FILENAME = "authentication-handoffs.json" +AUTHENTICATION_HANDOFF_SCHEMA_VERSION = "openadapt.capture.authentication-handoffs/v1" +AUTHENTICATION_METHODS = frozenset( + { + "password_manager", + "passkey", + "sso", + "mfa", + "device_unlock", + "other", + } +) +AUTHENTICATION_OUTCOMES = frozenset({"completed", "cancelled", "failed", "aborted"}) +SUPPRESSED_SOURCES = ( + "audio", + "browser", + "input", + "screen", + "structural", + "window", +) + +AuthenticationMethod = Literal[ + "password_manager", + "passkey", + "sso", + "mfa", + "device_unlock", + "other", +] +AuthenticationOutcome = Literal["completed", "cancelled", "failed", "aborted"] + + +class AuthenticationHandoffError(RuntimeError): + """An authentication handoff could not preserve its privacy boundary.""" + + +class AuthenticationBoundaryError(AuthenticationHandoffError): + """The recorder must fail because a source boundary became ambiguous.""" + + +class FreshFrameProof(BaseModel): + """Proof that Capture retained a new source frame before input resumed.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + timestamp: float = Field(ge=0) + source_ordinal: int = Field(ge=1) + frame_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + capture_source: str = Field(min_length=1, max_length=64) + window_geometry_generation: int | None = Field(default=None, ge=1) + + +class AuthenticationHandoff(BaseModel): + """One privacy-bounded authentication interval.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + interval_id: str = Field(min_length=36, max_length=36) + kind: Literal["authentication"] = "authentication" + methods: tuple[AuthenticationMethod, ...] = Field(min_length=1, max_length=6) + requires_user_presence: bool + saved_account_selected: bool + started_at: float = Field(ge=0) + entry_frame: FreshFrameProof | None = None + ended_at: float | None = Field(default=None, ge=0) + outcome: AuthenticationOutcome | None = None + suppressed_sources: tuple[ + Literal["audio", "browser", "input", "screen", "structural", "window"], + ..., + ] + resume_frame: FreshFrameProof | None = None + + @model_validator(mode="after") + def _closed_contract(self) -> "AuthenticationHandoff": + try: + uuid.UUID(self.interval_id) + except ValueError as exc: + raise ValueError("interval_id must be a UUID") from exc + if len(set(self.methods)) != len(self.methods): + raise ValueError("authentication methods must be unique") + if tuple(sorted(self.suppressed_sources)) != SUPPRESSED_SOURCES: + raise ValueError("the authentication handoff must suppress every sensitive source") + closed = self.ended_at is not None or self.outcome is not None + if closed and (self.ended_at is None or self.outcome is None): + raise ValueError("a closed authentication handoff needs an end time and outcome") + if self.ended_at is not None and self.ended_at < self.started_at: + raise ValueError("authentication handoff end time precedes its start time") + if self.entry_frame is not None and self.entry_frame.timestamp > self.started_at: + raise ValueError("authentication entry frame follows the protected start") + if self.outcome == "aborted": + if self.resume_frame is not None: + raise ValueError("an aborted authentication handoff cannot claim a resume frame") + elif self.outcome is not None and self.resume_frame is None: + raise ValueError("a closed authentication handoff needs a fresh resume frame") + if self.outcome is None and self.resume_frame is not None: + raise ValueError("an open authentication handoff cannot have a resume frame") + return self + + +class AuthenticationHandoffManifest(BaseModel): + """Canonical sealed list of authentication handoffs in one capture.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["openadapt.capture.authentication-handoffs/v1"] + intervals: tuple[AuthenticationHandoff, ...] + + @model_validator(mode="after") + def _ordered_nonoverlapping_intervals(self) -> "AuthenticationHandoffManifest": + ids = [interval.interval_id for interval in self.intervals] + if len(ids) != len(set(ids)): + raise ValueError("authentication handoff IDs must be unique") + previous_end: float | None = None + for interval in self.intervals: + if previous_end is not None and interval.started_at < previous_end: + raise ValueError("authentication handoffs cannot overlap") + previous_end = interval.ended_at + if previous_end is None and interval is not self.intervals[-1]: + raise ValueError("only the last authentication handoff can be open") + return self + + +@dataclass(frozen=True) +class AuthenticationHandoffHandle: + """Opaque owner handle for one active handoff.""" + + interval_id: str + + +class _RetentionLease: + """One in-flight sensitive-source operation that begin() must drain.""" + + def __init__( + self, + controller: "AuthenticationHandoffController", + *, + entry: bool = False, + resume: bool = False, + ) -> None: + self._controller = controller + self.entry = entry + self.resume = resume + self._released = False + + def release(self) -> None: + if self._released: + return + self._released = True + self._controller._release_retention() + + def __enter__(self) -> "_RetentionLease": + return self + + def __exit__(self, _exc_type, _exc, _tb) -> None: + self.release() + + +def _canonical_json_bytes(payload: object) -> bytes: + return ( + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + + +def _write_manifest(path: Path, manifest: AuthenticationHandoffManifest) -> None: + """Replace the owner-only marker file with canonical bytes.""" + + path.parent.mkdir(parents=True, exist_ok=True) + raw = _canonical_json_bytes(manifest.model_dump(mode="json")) + fd, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + if os.name != "nt": + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as output: + output.write(raw) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def load_authentication_handoffs( + capture_dir: str | Path, + *, + required: bool = False, +) -> AuthenticationHandoffManifest: + """Load and validate the canonical authentication marker sidecar.""" + + path = Path(capture_dir) / AUTHENTICATION_HANDOFF_FILENAME + if not path.exists(): + if required: + raise AuthenticationHandoffError("authentication handoff marker is missing") + return AuthenticationHandoffManifest( + schema_version=AUTHENTICATION_HANDOFF_SCHEMA_VERSION, + intervals=(), + ) + if not path.is_file() or path.is_symlink(): + raise AuthenticationHandoffError("authentication handoff marker is not a regular file") + raw = path.read_bytes() + try: + manifest = AuthenticationHandoffManifest.model_validate_json(raw) + except Exception as exc: + raise AuthenticationHandoffError("authentication handoff marker is malformed") from exc + if raw != _canonical_json_bytes(manifest.model_dump(mode="json")): + raise AuthenticationHandoffError("authentication handoff marker bytes are not canonical") + return manifest + + +def frame_sha256(image: object) -> str: + """Hash exact source pixels without retaining another image artifact.""" + + mode = str(getattr(image, "mode")) + width, height = getattr(image, "size") + pixels = getattr(image, "tobytes")() + header = _canonical_json_bytes({"height": height, "mode": mode, "width": width}) + return hashlib.sha256(b"openadapt.capture.frame.v1\0" + header + pixels).hexdigest() + + +class AuthenticationHandoffController: + """Coordinate atomic source suppression and fresh-frame resumption.""" + + def __init__(self) -> None: + self._condition = threading.Condition(threading.RLock()) + self._phase: Literal["unbound", "normal", "entering", "protected", "resuming"] = "unbound" + self._active_retentions = 0 + self._entry_capture_in_flight = False + self._entry_capture_complete = False + self._entry_frame_proof: FreshFrameProof | None = None + self._entry_error: BaseException | None = None + self._entry_frame_required = False + self._resume_capture_in_flight = False + self._path: Path | None = None + self._timestamp: Callable[[], float] | None = None + self._intervals: list[AuthenticationHandoff] = [] + self._active_interval_id: str | None = None + self._pending_outcome: AuthenticationOutcome | None = None + self._resume_error: BaseException | None = None + self._closed = False + # The audio process cannot share the thread condition. It can share this + # event and replaces protected chunks with generated silence. + self.audio_suppressed = multiprocessing.Event() + self.audio_suppression_ack = multiprocessing.Event() + self.audio_suppression_ack.set() + self._audio_enabled = False + + def configure_audio(self, enabled: bool) -> None: + """Declare whether begin() must wait for a microphone-process cut.""" + + with self._condition: + if self._phase != "unbound": + raise AuthenticationHandoffError( + "audio protection must be configured before recording starts" + ) + self._audio_enabled = bool(enabled) + if enabled: + self.audio_suppression_ack.clear() + else: + self.audio_suppression_ack.set() + + def configure_entry_frame(self, required: bool) -> None: + """Require a clean screen cut before a protected interval starts.""" + + with self._condition: + if self._phase != "unbound": + raise AuthenticationHandoffError( + "entry-frame protection must be configured before recording starts" + ) + self._entry_frame_required = bool(required) + + @property + def protected(self) -> bool: + """Return whether normal source retention is suppressed.""" + + with self._condition: + return self._phase in {"entering", "protected", "resuming"} + + def bind(self, capture_dir: str | Path, timestamp: Callable[[], float]) -> None: + """Bind the controller to a live capture and create its empty sidecar.""" + + with self._condition: + if self._phase != "unbound": + raise AuthenticationHandoffError("authentication controller is already bound") + self._path = Path(capture_dir) / AUTHENTICATION_HANDOFF_FILENAME + self._timestamp = timestamp + self._phase = "normal" + self._persist_locked() + self._condition.notify_all() + + def begin_retention(self) -> _RetentionLease | None: + """Enter one normal sensitive-source operation, or suppress it.""" + + with self._condition: + if self._phase != "normal" or self._closed: + return None + self._active_retentions += 1 + return _RetentionLease(self) + + def begin_screen_retention(self) -> _RetentionLease | None: + """Enter a normal frame operation or claim the one resume frame.""" + + with self._condition: + if self._closed or self._phase in {"unbound", "protected"}: + return None + if self._phase == "entering": + if not self._entry_frame_required: + return None + if self._entry_capture_in_flight or self._entry_capture_complete: + return None + self._entry_capture_in_flight = True + self._active_retentions += 1 + return _RetentionLease(self, entry=True) + if self._phase == "resuming": + if self._resume_capture_in_flight: + return None + self._resume_capture_in_flight = True + self._active_retentions += 1 + return _RetentionLease(self, resume=True) + self._active_retentions += 1 + return _RetentionLease(self) + + def _release_retention(self) -> None: + with self._condition: + if self._active_retentions <= 0: + raise RuntimeError("authentication retention lease underflow") + self._active_retentions -= 1 + self._condition.notify_all() + + def begin( + self, + *, + methods: AuthenticationMethod | Sequence[AuthenticationMethod], + requires_user_presence: bool, + saved_account_selected: bool = False, + timeout: float = 10.0, + interval_id: str | None = None, + ) -> AuthenticationHandoffHandle: + """Suppress all sensitive sources and persist an open handoff marker.""" + + normalized = self._normalize_methods(methods) + if interval_id is None: + interval_id = str(uuid.uuid4()) + else: + try: + interval_id = str(uuid.UUID(interval_id)) + except (AttributeError, TypeError, ValueError) as exc: + raise ValueError("authentication interval ID must be a UUID") from exc + deadline = time.monotonic() + timeout + with self._condition: + if self._closed or self._phase == "unbound": + raise AuthenticationHandoffError("recorder is not ready for authentication") + prior = next( + (interval for interval in self._intervals if interval.interval_id == interval_id), + None, + ) + if prior is not None: + if ( + prior.methods != normalized + or prior.requires_user_presence != requires_user_presence + or prior.saved_account_selected != saved_account_selected + ): + raise AuthenticationHandoffError( + "authentication interval ID was reused with different parameters" + ) + return AuthenticationHandoffHandle(interval_id) + if self._phase != "normal": + raise AuthenticationHandoffError("an authentication handoff is already active") + self._phase = "entering" + self._entry_capture_in_flight = False + self._entry_capture_complete = False + self._entry_frame_proof = None + self._entry_error = None + while self._active_retentions or ( + self._entry_frame_required and not self._entry_capture_complete + ): + if self._entry_error is not None or self._closed: + self._phase = "protected" + raise AuthenticationBoundaryError( + "authentication entry frame failed" + ) from self._entry_error + remaining = deadline - time.monotonic() + if remaining <= 0: + self._phase = "protected" + self._condition.notify_all() + raise AuthenticationBoundaryError( + "sensitive capture sources did not reach the protected boundary" + ) + self._condition.wait(min(remaining, 0.01)) + if self._audio_enabled: + self.audio_suppression_ack.clear() + self.audio_suppressed.set() + while not self.audio_suppression_ack.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + self._phase = "protected" + self._condition.notify_all() + raise AuthenticationBoundaryError("audio did not reach the protected boundary") + self._condition.wait(min(remaining, 0.01)) + started_at = self._now_locked() + interval = AuthenticationHandoff( + interval_id=interval_id, + methods=normalized, + requires_user_presence=requires_user_presence, + saved_account_selected=saved_account_selected, + started_at=started_at, + entry_frame=self._entry_frame_proof, + suppressed_sources=SUPPRESSED_SOURCES, + ) + self._intervals.append(interval) + self._active_interval_id = interval_id + self._entry_capture_in_flight = False + self._entry_capture_complete = False + self._entry_frame_proof = None + self._entry_error = None + self._phase = "protected" + try: + self._persist_locked() + except BaseException as exc: + self._intervals.pop() + self._active_interval_id = None + self._phase = "protected" + self._condition.notify_all() + raise AuthenticationBoundaryError( + "authentication start marker could not be persisted" + ) from exc + self._condition.notify_all() + return AuthenticationHandoffHandle(interval_id) + + def end( + self, + handle: AuthenticationHandoffHandle, + *, + outcome: Literal["completed", "cancelled", "failed"] = "completed", + timeout: float = 10.0, + ) -> AuthenticationHandoff: + """Request a fresh frame, then reopen normal source retention.""" + + if outcome not in {"completed", "cancelled", "failed"}: + raise ValueError("authentication outcome must be completed, cancelled, or failed") + deadline = time.monotonic() + timeout + with self._condition: + prior = next( + ( + interval + for interval in self._intervals + if interval.interval_id == handle.interval_id + ), + None, + ) + if prior is not None and prior.outcome is not None: + if prior.outcome != outcome: + raise AuthenticationHandoffError("authentication handoff outcome changed") + return prior + if handle.interval_id != self._active_interval_id: + raise AuthenticationHandoffError("authentication handoff handle is not active") + if self._phase == "protected": + self._pending_outcome = outcome + self._resume_error = None + self._phase = "resuming" + self._condition.notify_all() + elif self._phase != "resuming": + raise AuthenticationHandoffError("authentication handoff is not protected") + elif self._pending_outcome != outcome: + raise AuthenticationHandoffError("authentication handoff outcome changed") + while self._active_interval_id == handle.interval_id: + if self._resume_error is not None: + raise AuthenticationHandoffError( + "fresh-frame capture failed; the handoff remains protected" + ) from self._resume_error + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AuthenticationHandoffError( + "fresh-frame capture timed out; the handoff remains protected" + ) + self._condition.wait(remaining) + closed = self._intervals[-1] + if closed.outcome != outcome: + raise AuthenticationHandoffError( + "recorder stopped before the authentication handoff resumed" + ) + return closed + + def complete_resume_frame( + self, + lease: _RetentionLease, + *, + timestamp: float, + source_ordinal: int, + frame_sha256: str, + capture_source: str, + window_geometry_generation: int | None, + ) -> None: + """Persist the exact resume proof before normal input is reopened.""" + + if not lease.resume: + raise AuthenticationHandoffError("normal frame cannot complete a handoff") + with self._condition: + if self._phase != "resuming" or self._active_interval_id is None: + raise AuthenticationHandoffError("no authentication handoff awaits a frame") + proof = FreshFrameProof( + timestamp=timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256, + capture_source=capture_source, + window_geometry_generation=window_geometry_generation, + ) + active = self._intervals[-1] + closed = active.model_copy( + update={ + "ended_at": self._now_locked(), + "outcome": self._pending_outcome, + "resume_frame": proof, + } + ) + # Revalidate model_copy updates. Pydantic does not validate them. + closed = AuthenticationHandoff.model_validate(closed.model_dump(mode="json")) + self._intervals[-1] = closed + try: + self._persist_locked() + except BaseException as exc: + self._intervals[-1] = active + self._resume_error = exc + self._resume_capture_in_flight = False + self._condition.notify_all() + raise + self._active_interval_id = None + self._pending_outcome = None + self._resume_capture_in_flight = False + self._phase = "normal" + self.audio_suppressed.clear() + if self._audio_enabled: + self.audio_suppression_ack.clear() + self._condition.notify_all() + + def complete_entry_frame( + self, + lease: _RetentionLease, + *, + timestamp: float, + source_ordinal: int, + frame_sha256: str, + capture_source: str, + window_geometry_generation: int | None, + ) -> None: + """Acknowledge the clean frame that closes pre-handoff actions.""" + + if not lease.entry: + raise AuthenticationHandoffError("normal frame cannot open a handoff") + with self._condition: + if self._phase != "entering": + raise AuthenticationHandoffError("no authentication handoff awaits entry") + self._entry_frame_proof = FreshFrameProof( + timestamp=timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256, + capture_source=capture_source, + window_geometry_generation=window_geometry_generation, + ) + self._entry_capture_complete = True + self._condition.notify_all() + + def fail_boundary_frame(self, lease: _RetentionLease, error: BaseException) -> None: + """Fail an entry cut or retain resume protection after frame failure.""" + + if lease.entry: + with self._condition: + self._entry_error = error + self._entry_capture_in_flight = False + self._condition.notify_all() + return + self.fail_resume_frame(lease, error) + + def fail_resume_frame(self, lease: _RetentionLease, error: BaseException) -> None: + """Keep the boundary closed after a failed fresh-frame attempt.""" + + if not lease.resume: + return + with self._condition: + self._resume_error = error + self._resume_capture_in_flight = False + self._condition.notify_all() + + def abort_active(self) -> AuthenticationHandoff | None: + """Close an unfinished handoff for recorder shutdown without a resume claim.""" + + with self._condition: + self._closed = True + if self._active_interval_id is None: + self._condition.notify_all() + if self._phase in {"entering", "protected", "resuming"}: + raise AuthenticationBoundaryError( + "recording stopped inside an unmarked authentication boundary" + ) + return None + active = self._intervals[-1] + aborted = active.model_copy( + update={ + "ended_at": self._now_locked(), + "outcome": "aborted", + "resume_frame": None, + } + ) + aborted = AuthenticationHandoff.model_validate(aborted.model_dump(mode="json")) + self._intervals[-1] = aborted + self._persist_locked() + self._active_interval_id = None + self._pending_outcome = None + self._entry_capture_in_flight = False + self._entry_capture_complete = False + self._entry_frame_proof = None + self._resume_capture_in_flight = False + self._condition.notify_all() + return aborted + + def close(self) -> None: + """Prevent later handoffs after a recording stops.""" + + with self._condition: + self._closed = True + self._condition.notify_all() + + @staticmethod + def _normalize_methods( + methods: AuthenticationMethod | Sequence[AuthenticationMethod], + ) -> tuple[AuthenticationMethod, ...]: + if isinstance(methods, str): + values = (methods,) + else: + values = tuple(methods) + if not values or len(values) > 6: + raise ValueError("authentication methods must contain one to six values") + if any(not isinstance(value, str) for value in values): + raise ValueError("authentication methods are invalid or duplicated") + if len(values) != len(set(values)) or any( + value not in AUTHENTICATION_METHODS for value in values + ): + raise ValueError("authentication methods are invalid or duplicated") + return values # type: ignore[return-value] + + def _now_locked(self) -> float: + if self._timestamp is None: + raise AuthenticationHandoffError("authentication controller has no capture clock") + timestamp = float(self._timestamp()) + if not math.isfinite(timestamp) or timestamp < 0: + raise AuthenticationHandoffError("authentication timestamp is invalid") + return timestamp + + def _persist_locked(self) -> None: + if self._path is None: + raise AuthenticationHandoffError("authentication controller is not bound") + manifest = AuthenticationHandoffManifest( + schema_version=AUTHENTICATION_HANDOFF_SCHEMA_VERSION, + intervals=tuple(self._intervals), + ) + _write_manifest(self._path, manifest) diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index e6a4e80..bcacdfa 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -11,6 +11,12 @@ from pathlib import Path from typing import TYPE_CHECKING, Iterator +from openadapt_capture.authentication import ( + AuthenticationHandoff, + AuthenticationHandoffError, + frame_sha256, + load_authentication_handoffs, +) from openadapt_capture.browser_events import ( BoundingBox, BrowserClickEvent, @@ -894,6 +900,121 @@ def _validate_database_contract( elif capture.video_path is not None: raise InvalidCaptureEvent("sealed capture inventories an MP4 but claims no video frames") + try: + authentication_handoffs = capture.authentication_handoffs + except AuthenticationHandoffError as exc: + raise InvalidCaptureEvent(str(exc)) from exc + source_rows = [row for rows in rows_by_kind.values() for row in rows] + + def validate_authentication_frame(proof, label: str): + screenshot = screenshots_by_ordinal.get(proof.source_ordinal) + if screenshot is None or screenshot.timestamp != proof.timestamp: + raise InvalidCaptureEvent( + f"authentication {label} proof does not name a retained frame" + ) + if screenshot.png_data: + from PIL import Image + + with Image.open(io.BytesIO(screenshot.png_data)) as retained: + retained.load() + if frame_sha256(retained) != proof.frame_sha256: + raise InvalidCaptureEvent( + f"authentication {label} proof differs from retained pixels" + ) + if is_v2: + window_row = windows_rows_by_ordinal.get(proof.source_ordinal) + if window_row is None: + raise InvalidCaptureEvent( + f"window authentication {label} proof has no atomic geometry" + ) + state = WindowCaptureStateV2.model_validate(window_row.state) + if ( + proof.capture_source != state.capture_source + or proof.window_geometry_generation != state.geometry_generation + ): + raise InvalidCaptureEvent( + f"authentication {label} proof differs from its window frame" + ) + elif ( + proof.capture_source != "desktop-screenshot" + or proof.window_geometry_generation is not None + ): + raise InvalidCaptureEvent( + f"desktop authentication {label} proof has invalid source metadata" + ) + return screenshot + + for handoff in authentication_handoffs: + if handoff.outcome is None or handoff.ended_at is None: + raise InvalidCaptureEvent("sealed capture has an open authentication handoff") + entry = handoff.entry_frame + if entry is None: + raise InvalidCaptureEvent( + "authentication handoff has no clean entry-frame proof" + ) + entry_screenshot = validate_authentication_frame(entry, "entry") + if entry.timestamp > handoff.started_at: + raise InvalidCaptureEvent( + "authentication entry frame follows the protected start" + ) + if any( + row.source_ordinal > entry.source_ordinal + and row.timestamp <= handoff.started_at + for row in source_rows + ): + raise InvalidCaptureEvent( + "authentication entry proof is not the last pre-handoff frame" + ) + if handoff.outcome == "aborted": + if any( + row.source_ordinal > entry.source_ordinal + for row in source_rows + ): + raise InvalidCaptureEvent( + "sealed capture retained a sensitive source after an aborted " + "authentication handoff started" + ) + continue + proof = handoff.resume_frame + if proof is None: + raise InvalidCaptureEvent( + "closed authentication handoff has no fresh-frame proof" + ) + screenshot = validate_authentication_frame(proof, "resume") + if proof.source_ordinal <= entry.source_ordinal: + raise InvalidCaptureEvent( + "authentication resume frame does not follow its entry frame" + ) + if not (handoff.started_at <= proof.timestamp <= handoff.ended_at): + raise InvalidCaptureEvent( + "authentication resume frame is outside its protected interval" + ) + leaked_rows = [] + for row in source_rows: + if row is entry_screenshot: + continue + if row is screenshot: + continue + if ( + is_v2 + and row in rows_by_kind["window"] + and row.source_ordinal in {entry.source_ordinal, proof.source_ordinal} + and row.timestamp in {entry.timestamp, proof.timestamp} + ): + continue + if ( + entry.source_ordinal < row.source_ordinal <= proof.source_ordinal + or ( + row.source_ordinal > proof.source_ordinal + and row.timestamp <= proof.timestamp + ) + ): + leaked_rows.append(row) + if leaked_rows: + raise InvalidCaptureEvent( + "sealed capture retained a sensitive source inside an authentication handoff" + ) + list(capture.actions(include_moves=True)) @@ -1060,6 +1181,12 @@ def terminal(self): """Return the verified immutable terminal, or None for a legacy load.""" return self._verified_terminal + @property + def authentication_handoffs(self) -> tuple[AuthenticationHandoff, ...]: + """Return sealed source-suppression intervals without credential values.""" + + return load_authentication_handoffs(self.capture_dir).intervals + @property def id(self) -> str: """Capture ID.""" diff --git a/openadapt_capture/control.py b/openadapt_capture/control.py index 4ca75e5..d4754f7 100644 --- a/openadapt_capture/control.py +++ b/openadapt_capture/control.py @@ -34,6 +34,13 @@ import psutil +from openadapt_capture.authentication import ( + AuthenticationHandoff, + AuthenticationHandoffError, + AuthenticationHandoffHandle, + AuthenticationMethod, +) + CONTROL_SCHEMA_VERSION = "openadapt.capture-control.v1" TERMINAL_STATE_SCHEMA_VERSION = "openadapt.capture-terminal.v1" TERMINAL_STATE_FILENAME = "capture-state.json" @@ -69,6 +76,7 @@ class RecorderStatus: complete: bool integrity_verified: bool event_counts: dict[str, int] + authentication_protected: bool = False error_code: str | None = None failure_stage: str | None = None @@ -93,6 +101,7 @@ def from_payload(cls, payload: dict[str, Any]) -> "RecorderStatus": complete=payload["complete"] is True, integrity_verified=payload["integrity_verified"] is True, event_counts=counts, + authentication_protected=payload.get("authentication_protected") is True, error_code=( str(payload["error_code"]) if payload.get("error_code") is not None else None ), @@ -922,7 +931,8 @@ def _request( command: str, *, timeout: float, -) -> RecorderStatus: + payload: dict[str, Any] | None = None, +) -> dict[str, Any]: timeout = float(timeout) if not 0 < timeout <= _MAX_TIMEOUT_SECONDS: raise ValueError(f"timeout must be between 0 and {_MAX_TIMEOUT_SECONDS} seconds") @@ -937,6 +947,8 @@ def _request( "issued_at": time.time(), "timeout_seconds": timeout, } + if payload is not None: + request["payload"] = payload request["mac"] = _message_mac(descriptor.token, request) try: with socket.create_connection( @@ -980,6 +992,13 @@ def _request( if response.get("ok") is not True: error_code = str(response.get("error_code") or "control_request_failed") raise CaptureControlError(f"Capture control failed: {error_code}") + return response + + +def _status_from_response( + response: dict[str, Any], + descriptor: _ControlDescriptor, +) -> RecorderStatus: status = RecorderStatus.from_payload(response) if status.session_id != descriptor.session_id: raise CaptureControlAuthenticationError( @@ -997,7 +1016,10 @@ def status_recording( """Return the status of one exact active Capture session.""" descriptor = _select_descriptor(session_id, runtime_dir) - return _request(descriptor, "status", timeout=timeout) + return _status_from_response( + _request(descriptor, "status", timeout=timeout), + descriptor, + ) def stop_recording( @@ -1015,7 +1037,10 @@ def stop_recording( """ descriptor = _select_descriptor(session_id, runtime_dir) - status = _request(descriptor, "stop", timeout=timeout) + status = _status_from_response( + _request(descriptor, "stop", timeout=timeout), + descriptor, + ) if not status.complete or not status.integrity_verified or status.phase != "complete": raise CaptureControlError( f"Capture stop did not produce a verified complete session ({status.phase})." @@ -1023,6 +1048,76 @@ def stop_recording( return status +def begin_authentication_handoff( + *, + methods: AuthenticationMethod | tuple[AuthenticationMethod, ...], + requires_user_presence: bool, + saved_account_selected: bool = False, + interval_id: str | None = None, + session_id: str | None = None, + runtime_dir: str | os.PathLike[str] | None = None, + timeout: float = 10.0, +) -> AuthenticationHandoffHandle: + """Start a retry-safe protected interval in one exact recorder.""" + + if isinstance(methods, str): + normalized_methods = (methods,) + else: + normalized_methods = tuple(methods) + selected_interval_id = str(uuid.UUID(interval_id)) if interval_id else str(uuid.uuid4()) + descriptor = _select_descriptor(session_id, runtime_dir) + response = _request( + descriptor, + "authentication.begin", + timeout=timeout, + payload={ + "interval_id": selected_interval_id, + "methods": normalized_methods, + "requires_user_presence": requires_user_presence, + "saved_account_selected": saved_account_selected, + }, + ) + authentication = response.get("authentication") + if not isinstance(authentication, dict) or authentication.get( + "interval_id" + ) != selected_interval_id: + raise CaptureControlAuthenticationError( + "The recorder returned an invalid authentication handoff." + ) + return AuthenticationHandoffHandle(selected_interval_id) + + +def end_authentication_handoff( + handle: AuthenticationHandoffHandle, + *, + outcome: str = "completed", + session_id: str | None = None, + runtime_dir: str | os.PathLike[str] | None = None, + timeout: float = 10.0, +) -> AuthenticationHandoff: + """Retain a fresh frame and end one exact protected interval.""" + + descriptor = _select_descriptor(session_id, runtime_dir) + response = _request( + descriptor, + "authentication.end", + timeout=timeout, + payload={"interval_id": handle.interval_id, "outcome": outcome}, + ) + authentication = response.get("authentication") + try: + handoff = AuthenticationHandoff.model_validate(authentication) + except Exception as exc: + raise CaptureControlAuthenticationError( + "The recorder returned an invalid authentication handoff." + ) from exc + if handoff.interval_id != handle.interval_id or handoff.outcome != outcome: + raise CaptureControlAuthenticationError( + "The recorder returned a different authentication handoff." + ) + return handoff + + class _LoopbackServer(socketserver.ThreadingTCPServer): allow_reuse_address = False daemon_threads = False @@ -1071,6 +1166,8 @@ def __init__( capture_dir: str, snapshot: Callable[[], dict[str, Any]], stop: Callable[[float], dict[str, Any]], + begin_authentication: Callable[[dict[str, Any], float], dict[str, Any]] | None = None, + end_authentication: Callable[[dict[str, Any], float], dict[str, Any]] | None = None, session_id: str | None = None, runtime_dir: str | os.PathLike[str] | None = None, ) -> None: @@ -1080,6 +1177,8 @@ def __init__( self.process_started_at = psutil.Process(self.pid).create_time() self._snapshot = snapshot self._stop = stop + self._begin_authentication = begin_authentication + self._end_authentication = end_authentication self._runtime_dir_arg = runtime_dir self._token = secrets.token_urlsafe(48) self._server: _LoopbackServer | None = None @@ -1181,6 +1280,7 @@ def _response( ok: bool, status: dict[str, Any] | None = None, error_code: str | None = None, + authentication: dict[str, Any] | None = None, ) -> dict[str, Any]: response: dict[str, Any] = { "schema_version": CONTROL_SCHEMA_VERSION, @@ -1200,11 +1300,14 @@ def _response( "event_counts", "error_code", "failure_stage", + "authentication_protected", ): if key in status: response[key] = status[key] if error_code: response["error_code"] = error_code + if authentication is not None: + response["authentication"] = authentication response["mac"] = _message_mac(self._token, response) return response @@ -1222,6 +1325,14 @@ def _handle(self, connection: socket.socket) -> None: status = self._snapshot() elif command == "stop": status = self._stop(timeout) + elif command == "authentication.begin": + if self._begin_authentication is None: + raise CaptureControlAuthenticationError("unsupported_command") + status = self._begin_authentication(request.get("payload"), timeout) + elif command == "authentication.end": + if self._end_authentication is None: + raise CaptureControlAuthenticationError("unsupported_command") + status = self._end_authentication(request.get("payload"), timeout) else: raise CaptureControlAuthenticationError("unsupported_command") complete = status.get("complete") is True @@ -1240,7 +1351,12 @@ def _handle(self, connection: socket.socket) -> None: error_code=str(status.get("error_code") or "finalization_incomplete"), ) else: - response = self._response(request_id, ok=True, status=status) + response = self._response( + request_id, + ok=True, + status=status, + authentication=status.get("authentication"), + ) except CaptureControlAuthenticationError: # Do not reveal whether a token, session, or process field was wrong. response = self._response( @@ -1254,6 +1370,12 @@ def _handle(self, connection: socket.socket) -> None: ok=False, error_code="invalid_request", ) + except AuthenticationHandoffError: + response = self._response( + request_id, + ok=False, + error_code="authentication_handoff_failed", + ) try: connection.sendall(_canonical_json(response) + b"\n") except OSError: @@ -1290,7 +1412,9 @@ def __exit__(self, exc_type, exc_value, traceback) -> None: "CaptureControlError", "CaptureControlUnavailable", "RecorderStatus", + "begin_authentication_handoff", "discover_recorders", + "end_authentication_handoff", "status_recording", "stop_recording", ] diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 7876c26..fa9272d 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -45,6 +45,15 @@ from tqdm import tqdm from openadapt_capture import platform, utils, video, window +from openadapt_capture.authentication import ( + AuthenticationBoundaryError, + AuthenticationHandoff, + AuthenticationHandoffController, + AuthenticationHandoffError, + AuthenticationHandoffHandle, + AuthenticationMethod, + frame_sha256, +) from openadapt_capture.config import config from openadapt_capture.db import ( SQLITE_CAPTURE_JOURNAL_MODE, @@ -328,6 +337,19 @@ def fail(self, error: BaseException) -> None: self._reservation.fail(error) +@dataclass +class _GatedInputReservation: + """A native input reservation that holds the privacy boundary open.""" + + reservation: EventReservation | WindowActionReservation + retention: Any + + +@dataclass(frozen=True) +class _SuppressedInputReservation: + """Marker returned when a protected interval drops native input.""" + + class OrderedEventJournal: """A causal FIFO journal with pre-observation reservations.""" @@ -410,17 +432,18 @@ def reserve_window_action_receipt( raise return WindowActionReservation(reservation, window_scope, geometry) - def put(self, event: Event, block: bool = True, timeout: float | None = None) -> None: + def put(self, event: Event, block: bool = True, timeout: float | None = None) -> int: del block, timeout reservation = self.reserve(event.timestamp) reservation.complete(event) + return reservation.source_ordinal def commit_window_frame( self, event: Event, window_scope: WindowCaptureScope, generation: int, - ) -> None: + ) -> int: """Append one frame and publish its geometry in one critical section.""" timestamp = float(event.timestamp) if not math.isfinite(timestamp): @@ -442,6 +465,7 @@ def commit_window_frame( self._condition.notify_all() if failure is not None: raise failure + return entry.sequence def get(self, block: bool = True, timeout: float | None = None) -> Event: deadline = None if timeout is None else time.monotonic() + timeout @@ -1819,6 +1843,7 @@ def read_screen_events( input_frame_boundary: NativeInputFrameBoundary | None = None, terminal_frame_finished: threading.Event | None = None, terminal_frame_cancelled: threading.Event | None = None, + authentication: AuthenticationHandoffController | None = None, ) -> None: """Read screen events and add them to the event queue. @@ -1864,17 +1889,21 @@ def capture_one( seal_input: bool = False, ) -> tuple[float, float] | None: nonlocal started + retention = authentication.begin_screen_retention() if authentication else None + if authentication is not None and retention is None: + return None t_start = time.perf_counter() terminal_deadline = ( time.monotonic() + TERMINAL_FRAME_SEAL_TIMEOUT_SECONDS if seal_input else None ) - if window_scope is not None or desktop_scope is not None: - if seal_input and input_frame_boundary is None: - raise WindowCaptureError( - "terminal native capture requires the native input boundary" - ) + try: + if window_scope is not None or desktop_scope is not None: + if seal_input and input_frame_boundary is None: + raise WindowCaptureError( + "terminal native capture requires the native input boundary" + ) # Do not hold the observation boundary during pixel acquisition. # An OS input callback that arrives while the grab is in flight must # reserve and bind the previously published frame before this new @@ -1885,93 +1914,152 @@ def capture_one( # Any failed capture terminates the session. Retrying would omit a # frame while input continues and could produce complete-looking # evidence with a missing interval. - while True: - boundary_use = None - if input_frame_boundary is not None and require_input_boundary: + while True: + boundary_use = None + if input_frame_boundary is not None and require_input_boundary: + try: + boundary_use = input_frame_boundary.begin() + except _NativeFrameBoundaryClosed: + return None try: - boundary_use = input_frame_boundary.begin() - except _NativeFrameBoundaryClosed: - return None - try: - if window_scope is not None: - screenshot, _window_changed = window_scope.capture_frame( - publish=False - ) - else: - assert desktop_scope is not None + if window_scope is not None: + screenshot, _window_changed = window_scope.capture_frame( + publish=False + ) + else: + assert desktop_scope is not None # A monitor can move or change scale while the combined # frame keeps the same dimensions. Check both sides of # the grab so the pixels and input use one topology. - desktop_scope.assert_current(force=True) - screenshot = utils.take_screenshot() - desktop_scope.assert_current(force=True) - t_screenshot = time.perf_counter() - if screenshot is None: - raise WindowCaptureError("the captured screenshot was empty") - frame_timestamp = utils.get_timestamp() - if boundary_use is not None and not input_frame_boundary.finish( - boundary_use - ): - input_frame_boundary.complete(boundary_use) - boundary_use = None - if terminate_processing.is_set() and not seal_input: - return None - if ( - terminal_deadline is not None - and time.monotonic() >= terminal_deadline + desktop_scope.assert_current(force=True) + screenshot = utils.take_screenshot() + desktop_scope.assert_current(force=True) + t_screenshot = time.perf_counter() + if screenshot is None: + raise WindowCaptureError("the captured screenshot was empty") + frame_timestamp = utils.get_timestamp() + if boundary_use is not None and not input_frame_boundary.finish( + boundary_use ): + input_frame_boundary.complete(boundary_use) + boundary_use = None + if terminate_processing.is_set() and not seal_input: + return None + if ( + terminal_deadline is not None + and time.monotonic() >= terminal_deadline + ): + raise WindowCaptureError( + "native input did not become stable before the " + "terminal-frame deadline" + ) + if min_interval > 0: + if seal_input: + remaining = terminal_deadline - time.monotonic() + time.sleep(min(min_interval, max(0.0, remaining))) + else: + terminate_processing.wait(min_interval) + continue + if boundary_use is not None and seal_input: + input_frame_boundary.seal(boundary_use) + if not isinstance(event_q, OrderedEventJournal): raise WindowCaptureError( - "native input did not become stable before the " - "terminal-frame deadline" + "native-scoped capture requires the ordered event journal" ) - if min_interval > 0: - if seal_input: - remaining = terminal_deadline - time.monotonic() - time.sleep(min(min_interval, max(0.0, remaining))) - else: - terminate_processing.wait(min_interval) - continue - if boundary_use is not None and seal_input: - input_frame_boundary.seal(boundary_use) - if not isinstance(event_q, OrderedEventJournal): - raise WindowCaptureError( - "native-scoped capture requires the ordered event journal" - ) - if window_scope is not None: - generation = window_scope.current_generation() - scoped_frame = WindowScopedFrame( - image=screenshot, - window_event_data=window_scope.window_event_data(), - geometry_generation=generation, - ) - event_q.commit_window_frame( - Event(frame_timestamp, "screen", scoped_frame), - window_scope, - generation, - ) - else: - event_q.put(Event(frame_timestamp, "screen", screenshot)) - if not started: - started_event.set() - started = True - return t_start, t_screenshot - finally: - if boundary_use is not None: - input_frame_boundary.complete(boundary_use) - screenshot = utils.take_screenshot() - t_screenshot = time.perf_counter() - if screenshot is None: - raise WindowCaptureError("the captured screenshot was empty") - if not started: - started_event.set() - started = True - frame_timestamp = utils.get_timestamp() - event_q.put(Event(frame_timestamp, "screen", screenshot)) - return t_start, t_screenshot + if window_scope is not None: + generation = window_scope.current_generation() + scoped_frame = WindowScopedFrame( + image=screenshot, + window_event_data=window_scope.window_event_data(), + geometry_generation=generation, + ) + source_ordinal = event_q.commit_window_frame( + Event(frame_timestamp, "screen", scoped_frame), + window_scope, + generation, + ) + capture_source = str( + scoped_frame.window_event_data.get("state", {}).get( + "capture_source", "platform-window-image" + ) + ) + else: + generation = None + source_ordinal = event_q.put( + Event(frame_timestamp, "screen", screenshot) + ) + capture_source = "desktop-screenshot" + if retention is not None and retention.resume: + authentication.complete_resume_frame( + retention, + timestamp=frame_timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256(screenshot), + capture_source=capture_source, + window_geometry_generation=generation, + ) + elif retention is not None and retention.entry: + authentication.complete_entry_frame( + retention, + timestamp=frame_timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256(screenshot), + capture_source=capture_source, + window_geometry_generation=generation, + ) + if not started: + started_event.set() + started = True + return t_start, t_screenshot + finally: + if boundary_use is not None: + input_frame_boundary.complete(boundary_use) + screenshot = utils.take_screenshot() + t_screenshot = time.perf_counter() + if screenshot is None: + raise WindowCaptureError("the captured screenshot was empty") + if not started: + started_event.set() + started = True + frame_timestamp = utils.get_timestamp() + source_ordinal = event_q.put(Event(frame_timestamp, "screen", screenshot)) + if retention is not None and retention.resume: + authentication.complete_resume_frame( + retention, + timestamp=frame_timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256(screenshot), + capture_source="desktop-screenshot", + window_geometry_generation=None, + ) + elif retention is not None and retention.entry: + authentication.complete_entry_frame( + retention, + timestamp=frame_timestamp, + source_ordinal=source_ordinal, + frame_sha256=frame_sha256(screenshot), + capture_source="desktop-screenshot", + window_geometry_generation=None, + ) + return t_start, t_screenshot + except BaseException as exc: + if authentication is not None and retention is not None: + authentication.fail_boundary_frame(retention, exc) + raise + finally: + if retention is not None: + retention.release() while not terminate_processing.is_set(): timing = capture_one() if timing is None: + if ( + authentication is not None + and authentication.protected + and not terminate_processing.is_set() + ): + terminate_processing.wait(min(0.05, min_interval or 0.05)) + continue break t_start, t_screenshot = timing # Throttle: sleep for the remainder of the frame interval @@ -1984,6 +2072,9 @@ def capture_one( t_end = time.perf_counter() _screen_timing.append((t_screenshot - t_start, t_end - t_start)) + if authentication is not None and authentication.protected: + if terminal_frame_cancelled is not None: + terminal_frame_cancelled.set() terminal_cancelled = ( terminal_frame_cancelled is not None and terminal_frame_cancelled.is_set() ) @@ -2002,8 +2093,12 @@ def capture_one( t_start, t_screenshot = timing t_end = time.perf_counter() _screen_timing.append((t_screenshot - t_start, t_end - t_start)) - elif (window_scope is not None or desktop_scope is not None) and ( + elif ( + not (authentication is not None and authentication.protected) + and (window_scope is not None or desktop_scope is not None) + and ( input_finished is not None + ) ): input_finished.wait() timing = capture_one(require_input_boundary=False) @@ -2020,6 +2115,7 @@ def read_window_events( terminate_processing: multiprocessing.Event, recording: Recording, started_event: threading.Event, + authentication: AuthenticationHandoffController | None = None, ) -> None: """Read window events and add them to the event queue. @@ -2038,38 +2134,40 @@ def read_window_events( prev_window_data = {} started = False while not terminate_processing.is_set(): - window_data = window.get_active_window_data() - if not window_data: + retention = authentication.begin_retention() if authentication is not None else None + if authentication is not None and retention is None: time.sleep(0.1) continue + try: + window_data = window.get_active_window_data() + if not window_data: + time.sleep(0.1) + continue - if not started: - started_event.set() - started = True + if not started: + started_event.set() + started = True - if window_data["title"] != prev_window_data.get("title") or window_data[ - "window_id" - ] != prev_window_data.get("window_id"): - # TODO: fix exception sometimes triggered by the next line on win32: - # File "\Python39\lib\threading.py" line 917, in run - # File "...\openadapt\record.py", line 277, in read window events - # File "...\env\lib\site-packages\loguru\logger.py" line 1977, in info - # File "...\env\lib\site-packages\loguru\_logger.py", line 1964, in _log - # for handler in core.handlers.values): - # RuntimeError: dictionary changed size during iteration - _window_data = window_data - _window_data.pop("state") - logger.info(f"{_window_data=}") - if window_data != prev_window_data: - logger.debug("Queuing window event for writing") - event_q.put( - Event( - utils.get_timestamp(), - "window", - window_data, + if window_data["title"] != prev_window_data.get("title") or window_data[ + "window_id" + ] != prev_window_data.get("window_id"): + # Log a copy. The retained event still needs its state field. + _window_data = dict(window_data) + _window_data.pop("state", None) + logger.info(f"{_window_data=}") + if window_data != prev_window_data: + logger.debug("Queuing window event for writing") + event_q.put( + Event( + utils.get_timestamp(), + "window", + window_data, + ) ) - ) - prev_window_data = window_data + prev_window_data = window_data + finally: + if retention is not None: + retention.release() time.sleep(0.1) # poll ~10 times/sec instead of tight loop @@ -2264,6 +2362,7 @@ def read_input_events( input_frame_boundary: NativeInputFrameBoundary | None = None, terminal_frame_finished: threading.Event | None = None, terminal_frame_cancelled: threading.Event | None = None, + authentication: AuthenticationHandoffController | None = None, ) -> None: """Read globally ordered keyboard and mouse events from one native observer.""" stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] @@ -2344,25 +2443,66 @@ def on_observed( logger.info("Stop sequence entered! Stopping recording now.") stop_sequence_detected = True + def retained_on_observed( + event: ObservedInput, + reservation: object | None = None, + ) -> None: + """Drop protected input before structural observation or persistence.""" + + if isinstance(reservation, _SuppressedInputReservation): + return + if isinstance(reservation, _GatedInputReservation): + try: + on_observed(event, reservation.reservation) + finally: + reservation.retention.release() + return + retention = authentication.begin_retention() if authentication is not None else None + if authentication is not None and retention is None: + return + try: + on_observed(event, reservation) + finally: + if retention is not None: + retention.release() + if structural_observer is not None: start_hook = getattr(structural_observer, "open_current_thread", None) stop_hook = getattr(structural_observer, "close_current_thread", None) if callable(start_hook): - setattr(on_observed, "_openadapt_delivery_thread_start", start_hook) + setattr(retained_on_observed, "_openadapt_delivery_thread_start", start_hook) if callable(stop_hook): - setattr(on_observed, "_openadapt_delivery_thread_stop", stop_hook) + setattr(retained_on_observed, "_openadapt_delivery_thread_stop", stop_hook) if isinstance(event_q, OrderedEventJournal): def reserve_observed(timestamp: float): - if isinstance(coordinate_scope, WindowCaptureScope): - return event_q.reserve_window_action_receipt( - timestamp, - coordinate_scope, - ) - return event_q.reserve(timestamp) + retention = authentication.begin_retention() if authentication is not None else None + if authentication is not None and retention is None: + return _SuppressedInputReservation() + try: + if isinstance(coordinate_scope, WindowCaptureScope): + reservation = event_q.reserve_window_action_receipt( + timestamp, + coordinate_scope, + ) + else: + reservation = event_q.reserve(timestamp) + except BaseException: + if retention is not None: + retention.release() + raise + if retention is None: + return reservation + return _GatedInputReservation(reservation, retention) def deliver_observed(event: ObservedInput, reservation: object) -> None: + if isinstance( + reservation, + (_GatedInputReservation, _SuppressedInputReservation), + ): + retained_on_observed(event, reservation) + return if not isinstance( reservation, (EventReservation, WindowActionReservation), @@ -2370,17 +2510,17 @@ def deliver_observed(event: ObservedInput, reservation: object) -> None: raise EventJournalOrderingError( "native input delivery received an invalid source reservation" ) - on_observed(event, reservation) + retained_on_observed(event, reservation) - setattr(on_observed, "_openadapt_input_receipt", reserve_observed) - setattr(on_observed, "_openadapt_input_delivery", deliver_observed) + setattr(retained_on_observed, "_openadapt_input_receipt", reserve_observed) + setattr(retained_on_observed, "_openadapt_input_delivery", deliver_observed) observer = None started = False observer_failed = False try: observer = create_input_observer( - on_observed, + retained_on_observed, observe_keyboard=True, observe_mouse=True, capture_mouse_moves=True, @@ -2452,6 +2592,8 @@ def record_audio( db_path: str, terminate_processing: multiprocessing.Event, started_event: multiprocessing.Event, + authentication_suppressed: Any = None, + authentication_suppression_ack: Any = None, ) -> None: """Record audio narration during the recording and store data in database. @@ -2486,6 +2628,7 @@ def record_audio( signal.signal(signal.SIGINT, signal.SIG_IGN) audio_frames = [] # to store audio frames + callback_boundary = threading.Lock() import sounddevice @@ -2497,8 +2640,21 @@ def audio_callback( Note: time is of type cffi.FFI.CData, but since we don't use this argument and we also don't use the cffi library, the Any type annotation is used. """ - # called whenever there is new audio frames - audio_frames.append(indata.copy()) + # Preserve the audio clock with generated silence. Never retain the + # protected waveform. Check both sides of the copy so a boundary that + # arrives during the callback cannot commit the copied samples. + with callback_boundary: + suppressed_before = ( + authentication_suppressed is not None + and authentication_suppressed.is_set() + ) + captured = np.zeros_like(indata) if suppressed_before else indata.copy() + if ( + authentication_suppressed is not None + and authentication_suppressed.is_set() + ): + captured.fill(0) + audio_frames.append(captured) # open InputStream and start recording while ActionEvents are recorded audio_stream = sounddevice.InputStream(callback=audio_callback, samplerate=16000, channels=1) @@ -2510,7 +2666,16 @@ def audio_callback( # TODO: handle race condition, e.g. by sending synthetic events from main thread started_event.set() - terminate_processing.wait() + while not terminate_processing.wait(0.01): + if authentication_suppressed is None or authentication_suppression_ack is None: + continue + if authentication_suppressed.is_set(): + # Acquiring this lock proves that every callback which entered + # before suppression has finished appending its pre-boundary chunk. + with callback_boundary: + authentication_suppression_ack.set() + else: + authentication_suppression_ack.clear() audio_stream.stop() audio_stream.close() @@ -2629,6 +2794,7 @@ def record( window_title: str | None = None, structural_observer: StructuralObserver | None = None, child_registry: dict[str, Any] | None = None, + authentication: AuthenticationHandoffController | None = None, ) -> int | None: """Record native screenshots, action events, and window events. @@ -2745,6 +2911,11 @@ def record( ), ) recording_timestamp = recording.timestamp + if authentication is None: + authentication = AuthenticationHandoffController() + authentication.configure_audio(bool(config.RECORD_AUDIO)) + authentication.configure_entry_frame(True) + authentication.bind(capture_dir, utils.get_timestamp) # create_recording() established the one shared clock epoch for this # capture. Every thread producer inherits that epoch. A thread must not @@ -2846,6 +3017,7 @@ def record( terminate_processing, recording, task_started_events.setdefault("window_event_reader", threading.Event()), + authentication, ), terminate_processing, task_errors, @@ -2872,6 +3044,7 @@ def record( input_frame_boundary, terminal_frame_finished, terminal_frame_cancelled, + authentication, ), terminate_processing, task_errors, @@ -2891,6 +3064,7 @@ def record( input_frame_boundary, terminal_frame_finished, terminal_frame_cancelled, + authentication, ) input_event_reader = threading.Thread( target=_run_task_fail_loud, @@ -3052,6 +3226,8 @@ def record( db_path, terminate_processing, task_started_events.setdefault("audio_event_writer", multiprocessing.Event()), + authentication.audio_suppressed, + authentication.audio_suppression_ack, ), ) audio_recorder.start() @@ -3170,6 +3346,11 @@ def record( timeout=pre_ready_timeout, ) + # An owner can stop while an attended handoff is still open. Retain an + # explicit aborted interval. Do not capture a sensitive terminal frame. + authentication.abort_active() + authentication.close() + # No writer can stop while the event processor can still enqueue work. # Signal writer completion only after all producers have exited. terminate_writers.set() @@ -3441,6 +3622,7 @@ def __init__( self._worker_error: BaseException | None = None self._worker_error_lock = threading.Lock() self._structural_observer = structural_observer + self._authentication = AuthenticationHandoffController() self._control_server = None self._control_state_lock = threading.RLock() self._control_stop_lock = threading.Lock() @@ -3480,6 +3662,7 @@ def _control_payload(self) -> dict[str, Any]: "browser": self._num_browser_events.value, "video": self._num_video_events.value, }, + "authentication_protected": self._authentication.protected, } def _persist_control_state(self) -> None: @@ -3727,11 +3910,73 @@ def _start_control_server(self) -> None: capture_dir=self.capture_dir, snapshot=self._control_payload, stop=self._control_stop, + begin_authentication=self._control_begin_authentication, + end_authentication=self._control_end_authentication, session_id=self._control_session_id, runtime_dir=self._control_runtime_dir, ) self._control_server = server.start() + def _control_begin_authentication( + self, + payload: dict[str, Any], + timeout: float, + ) -> dict[str, Any]: + """Validate and execute one authenticated cross-process begin request.""" + + if not isinstance(payload, dict) or set(payload) != { + "interval_id", + "methods", + "requires_user_presence", + "saved_account_selected", + }: + raise ValueError("invalid authentication begin payload") + if not isinstance(payload["methods"], (list, tuple)): + raise ValueError("authentication methods must be a list") + if not isinstance(payload["interval_id"], str) or not all( + isinstance(method, str) for method in payload["methods"] + ): + raise ValueError("authentication begin identifiers must be strings") + if not isinstance(payload["requires_user_presence"], bool) or not isinstance( + payload["saved_account_selected"], bool + ): + raise ValueError("authentication handoff flags must be booleans") + handle = self.begin_authentication( + methods=tuple(payload["methods"]), + requires_user_presence=payload["requires_user_presence"], + saved_account_selected=payload["saved_account_selected"], + timeout=timeout, + interval_id=payload["interval_id"], + ) + return { + **self._control_payload(), + "authentication": {"interval_id": handle.interval_id}, + } + + def _control_end_authentication( + self, + payload: dict[str, Any], + timeout: float, + ) -> dict[str, Any]: + """Validate and execute one authenticated cross-process end request.""" + + if not isinstance(payload, dict) or set(payload) != {"interval_id", "outcome"}: + raise ValueError("invalid authentication end payload") + if not isinstance(payload["interval_id"], str) or not isinstance( + payload["outcome"], str + ): + raise ValueError("authentication end identifiers must be strings") + interval_id = str(uuid.UUID(str(payload["interval_id"]))) + handoff = self.end_authentication( + AuthenticationHandoffHandle(interval_id), + outcome=payload["outcome"], + timeout=timeout, + ) + return { + **self._control_payload(), + "authentication": handoff.model_dump(mode="json"), + } + def _control_stop(self, timeout: float) -> dict[str, Any]: """Idempotently request stop and wait for the one finalization result.""" with self._control_stop_lock: @@ -3807,6 +4052,7 @@ def _run_record(self) -> None: send_profile=self._send_profile, structural_observer=self._structural_observer, child_registry=self._child_registry, + authentication=self._authentication, ) if last_source_ordinal is not None: self._last_source_ordinal = last_source_ordinal @@ -3978,6 +4224,65 @@ def wait_for_ready(self, timeout: float = 60) -> bool: self.check_health() return self._ready_event.is_set() + def begin_authentication( + self, + *, + methods: AuthenticationMethod | tuple[AuthenticationMethod, ...], + requires_user_presence: bool, + saved_account_selected: bool = False, + timeout: float = 10.0, + interval_id: str | None = None, + ) -> AuthenticationHandoffHandle: + """Start an attended authentication interval with source suppression. + + The method accepts bounded classes only. It does not accept a provider, + account identifier, password, OTP, recovery code, or free text. + """ + + if not self._ready_event.is_set() or not self.is_recording: + raise AuthenticationHandoffError( + "recorder must be ready before authentication begins" + ) + try: + return self._authentication.begin( + methods=methods, + requires_user_presence=requires_user_presence, + saved_account_selected=saved_account_selected, + timeout=timeout, + interval_id=interval_id, + ) + except AuthenticationBoundaryError as exc: + self._set_worker_error(exc) + self._terminate_processing.set() + raise + + def end_authentication( + self, + handle: AuthenticationHandoffHandle, + *, + outcome: str = "completed", + timeout: float = 10.0, + ) -> AuthenticationHandoff: + """Retain a fresh exact frame, then resume normal source capture.""" + + if not self.is_recording: + raise AuthenticationHandoffError( + "recorder stopped before authentication could resume" + ) + if outcome not in {"completed", "cancelled", "failed"}: + raise ValueError("authentication outcome must be completed, cancelled, or failed") + return self._authentication.end( + handle, + outcome=outcome, # type: ignore[arg-type] + timeout=timeout, + ) + + @property + def authentication_protected(self) -> bool: + """Whether Capture is suppressing sensitive sources.""" + + return self._authentication.protected + @property def is_recording(self) -> bool: """Whether recording is currently active.""" diff --git a/tests/test_authentication_handoff.py b/tests/test_authentication_handoff.py new file mode 100644 index 0000000..f7b3e05 --- /dev/null +++ b/tests/test_authentication_handoff.py @@ -0,0 +1,499 @@ +"""Source-time authentication handoff contract tests.""" + +from __future__ import annotations + +import io +import itertools +import json +import multiprocessing +import queue +import threading +import time + +import pytest +from PIL import Image + +from openadapt_capture import recorder as recorder_module +from openadapt_capture.authentication import ( + AUTHENTICATION_HANDOFF_FILENAME, + AuthenticationBoundaryError, + AuthenticationHandoff, + AuthenticationHandoffController, + AuthenticationHandoffError, + AuthenticationHandoffManifest, + FreshFrameProof, + _write_manifest, + frame_sha256, + load_authentication_handoffs, +) +from openadapt_capture.capture import CaptureSession, InvalidCaptureEvent +from openadapt_capture.config import RecordingConfig, config_override +from openadapt_capture.input_observer import ObservedMouseButton +from openadapt_capture.recorder import OrderedEventJournal, read_screen_events +from openadapt_capture.terminal import ( + ARTIFACT_MANIFEST_FILENAME, + CAPTURE_TERMINAL_FILENAME, + seal_capture, +) +from tests.test_capture_terminal import _desktop_capture_directory + + +class _Clock: + def __init__(self) -> None: + self.value = 1.0 + + def __call__(self) -> float: + self.value += 1.0 + return self.value + + +def _bound_controller(tmp_path): + controller = AuthenticationHandoffController() + controller.bind(tmp_path, _Clock()) + return controller + + +def test_empty_manifest_is_created_with_owner_only_permissions(tmp_path) -> None: + _bound_controller(tmp_path) + + marker = tmp_path / AUTHENTICATION_HANDOFF_FILENAME + assert load_authentication_handoffs(tmp_path).intervals == () + assert marker.stat().st_mode & 0o077 == 0 + + +def test_begin_drains_inflight_retention_before_marker_starts(tmp_path) -> None: + controller = _bound_controller(tmp_path) + retention = controller.begin_retention() + assert retention is not None + result = [] + + thread = threading.Thread( + target=lambda: result.append( + controller.begin( + methods="password_manager", + requires_user_presence=True, + saved_account_selected=True, + ) + ) + ) + thread.start() + time.sleep(0.05) + + assert controller.protected is True + assert controller.audio_suppressed.is_set() is False + assert result == [] + assert controller.begin_retention() is None + + retention.release() + thread.join(timeout=1) + assert len(result) == 1 + interval = load_authentication_handoffs(tmp_path).intervals[0] + assert interval.methods == ("password_manager",) + assert interval.saved_account_selected is True + assert interval.outcome is None + + +def test_begin_waits_for_audio_process_acknowledgement(tmp_path) -> None: + controller = AuthenticationHandoffController() + controller.configure_audio(True) + controller.bind(tmp_path, _Clock()) + result = [] + thread = threading.Thread( + target=lambda: result.append( + controller.begin(methods="passkey", requires_user_presence=True) + ) + ) + thread.start() + time.sleep(0.05) + + assert result == [] + assert controller.audio_suppressed.is_set() + controller.audio_suppression_ack.set() + thread.join(timeout=1) + + assert len(result) == 1 + + +def test_begin_barrier_timeout_stays_protected_and_fails_closed(tmp_path) -> None: + controller = AuthenticationHandoffController() + controller.configure_audio(True) + controller.bind(tmp_path, _Clock()) + + with pytest.raises(AuthenticationBoundaryError, match="protected boundary"): + controller.begin( + methods="passkey", + requires_user_presence=True, + timeout=0.01, + ) + + assert controller.protected is True + assert controller.audio_suppressed.is_set() + assert load_authentication_handoffs(tmp_path).intervals == () + with pytest.raises(AuthenticationBoundaryError, match="unmarked"): + controller.abort_active() + + +def test_end_waits_for_fresh_frame_before_reopening_sources(tmp_path) -> None: + controller = _bound_controller(tmp_path) + handle = controller.begin( + methods=("password_manager", "mfa"), + requires_user_presence=True, + ) + result = [] + thread = threading.Thread( + target=lambda: result.append(controller.end(handle, outcome="completed", timeout=1)) + ) + thread.start() + time.sleep(0.05) + + assert controller.begin_retention() is None + resume = controller.begin_screen_retention() + assert resume is not None and resume.resume + image = Image.new("RGB", (2, 2), "green") + controller.complete_resume_frame( + resume, + timestamp=3.0, + source_ordinal=7, + frame_sha256=frame_sha256(image), + capture_source="desktop-screenshot", + window_geometry_generation=None, + ) + resume.release() + thread.join(timeout=1) + + assert len(result) == 1 + assert result[0].outcome == "completed" + assert result[0].resume_frame.source_ordinal == 7 + assert controller.audio_suppressed.is_set() is False + normal = controller.begin_retention() + assert normal is not None + normal.release() + + +def test_resume_timeout_keeps_capture_protected(tmp_path) -> None: + controller = _bound_controller(tmp_path) + handle = controller.begin(methods="passkey", requires_user_presence=True) + + with pytest.raises(AuthenticationHandoffError, match="remains protected"): + controller.end(handle, timeout=0.01) + + assert controller.protected is True + assert controller.begin_retention() is None + + +def test_shutdown_records_aborted_handoff_without_resume_claim(tmp_path) -> None: + controller = _bound_controller(tmp_path) + controller.begin(methods="sso", requires_user_presence=False) + + aborted = controller.abort_active() + + assert aborted is not None + assert aborted.outcome == "aborted" + assert aborted.resume_frame is None + assert load_authentication_handoffs(tmp_path).intervals == (aborted,) + + +@pytest.mark.parametrize( + "methods", + [ + (), + ("password_manager", "password_manager"), + ({"provider": "vault"},), + "1password", + "password", + ], +) +def test_method_contract_rejects_identity_or_secret_free_text(tmp_path, methods) -> None: + controller = _bound_controller(tmp_path) + + with pytest.raises(ValueError, match="methods"): + controller.begin(methods=methods, requires_user_presence=True) + + +def test_marker_loader_rejects_noncanonical_or_extra_data(tmp_path) -> None: + _bound_controller(tmp_path) + marker = tmp_path / AUTHENTICATION_HANDOFF_FILENAME + payload = json.loads(marker.read_text()) + payload["account"] = "person@example.test" + marker.write_text(json.dumps(payload)) + + with pytest.raises(AuthenticationHandoffError, match="malformed"): + load_authentication_handoffs(tmp_path) + + +def test_sealed_loader_binds_resume_proof_to_exact_retained_frame(tmp_path) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(1, 2), + action_ordinal=None, + ) + (capture_dir / ARTIFACT_MANIFEST_FILENAME).unlink() + (capture_dir / CAPTURE_TERMINAL_FILENAME).unlink() + with CaptureSession.load(capture_dir) as capture: + digests = [] + for screenshot in capture._recording.screenshots: + with Image.open(io.BytesIO(screenshot.png_data)) as retained: + retained.load() + digests.append(frame_sha256(retained)) + handoff = AuthenticationHandoff( + interval_id="00000000-0000-4000-8000-000000000001", + methods=("password_manager",), + requires_user_presence=True, + saved_account_selected=True, + started_at=11.5, + entry_frame=FreshFrameProof( + timestamp=11.0, + source_ordinal=1, + frame_sha256=digests[0], + capture_source="desktop-screenshot", + ), + ended_at=12.5, + outcome="completed", + suppressed_sources=( + "audio", + "browser", + "input", + "screen", + "structural", + "window", + ), + resume_frame=FreshFrameProof( + timestamp=12.0, + source_ordinal=2, + frame_sha256=digests[1], + capture_source="desktop-screenshot", + ), + ) + _write_manifest( + capture_dir / AUTHENTICATION_HANDOFF_FILENAME, + AuthenticationHandoffManifest( + schema_version="openadapt.capture.authentication-handoffs/v1", + intervals=(handoff,), + ), + ) + seal_capture( + capture_dir, + session_id="authentication-test", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={ + "action": 0, + "screen": 2, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=2, + ) + + with CaptureSession.load_verified(capture_dir) as capture: + assert capture.authentication_handoffs == (handoff,) + + (capture_dir / ARTIFACT_MANIFEST_FILENAME).unlink() + (capture_dir / CAPTURE_TERMINAL_FILENAME).unlink() + wrong = handoff.model_copy( + update={"resume_frame": handoff.resume_frame.model_copy(update={"source_ordinal": 1})} + ) + _write_manifest( + capture_dir / AUTHENTICATION_HANDOFF_FILENAME, + AuthenticationHandoffManifest( + schema_version="openadapt.capture.authentication-handoffs/v1", + intervals=(wrong,), + ), + ) + seal_capture( + capture_dir, + session_id="authentication-test", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={ + "action": 0, + "screen": 2, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=2, + ) + with pytest.raises(InvalidCaptureEvent, match="resume proof"): + CaptureSession.load_verified(capture_dir) + + +def test_sealed_loader_rejects_source_event_after_aborted_handoff(tmp_path) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(1, 2), + action_ordinal=None, + ) + (capture_dir / ARTIFACT_MANIFEST_FILENAME).unlink() + (capture_dir / CAPTURE_TERMINAL_FILENAME).unlink() + with CaptureSession.load(capture_dir) as capture: + first = capture._recording.screenshots[0] + with Image.open(io.BytesIO(first.png_data)) as retained: + retained.load() + entry_digest = frame_sha256(retained) + handoff = AuthenticationHandoff( + interval_id="00000000-0000-4000-8000-000000000002", + methods=("sso",), + requires_user_presence=False, + saved_account_selected=False, + started_at=11.5, + entry_frame=FreshFrameProof( + timestamp=11.0, + source_ordinal=1, + frame_sha256=entry_digest, + capture_source="desktop-screenshot", + ), + ended_at=12.5, + outcome="aborted", + suppressed_sources=( + "audio", + "browser", + "input", + "screen", + "structural", + "window", + ), + ) + _write_manifest( + capture_dir / AUTHENTICATION_HANDOFF_FILENAME, + AuthenticationHandoffManifest( + schema_version="openadapt.capture.authentication-handoffs/v1", + intervals=(handoff,), + ), + ) + seal_capture( + capture_dir, + session_id="authentication-abort-test", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={ + "action": 0, + "screen": 2, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=2, + ) + + with pytest.raises(InvalidCaptureEvent, match="after an aborted"): + CaptureSession.load_verified(capture_dir) + + +def test_screen_reader_captures_nothing_until_resume_frame(tmp_path, monkeypatch) -> None: + timestamps = itertools.count(10) + + def timestamp() -> float: + return float(next(timestamps)) + + controller = AuthenticationHandoffController() + controller.configure_entry_frame(True) + controller.bind(tmp_path, timestamp) + event_q = OrderedEventJournal() + terminate = multiprocessing.Event() + started = threading.Event() + calls = 0 + + def screenshot() -> Image.Image: + nonlocal calls + calls += 1 + return Image.new("RGB", (4, 4), (calls % 255, 0, 0)) + + monkeypatch.setattr("openadapt_capture.recorder.utils.take_screenshot", screenshot) + monkeypatch.setattr( + "openadapt_capture.recorder.utils.get_timestamp", + timestamp, + ) + with config_override(RecordingConfig(screen_capture_fps=100)): + reader = threading.Thread( + target=read_screen_events, + args=(event_q, terminate, object(), started), + kwargs={"authentication": controller}, + ) + reader.start() + assert started.wait(timeout=1) + event_q.get(timeout=1) + handle = controller.begin( + methods="password_manager", + requires_user_presence=True, + ) + entry_events = [] + while True: + try: + entry_events.append(event_q.get_nowait()) + except queue.Empty: + break + assert entry_events + entry_ordinal = entry_events[-1].source_ordinal + protected_calls = calls + time.sleep(0.05) + assert calls == protected_calls + + result = [] + closer = threading.Thread(target=lambda: result.append(controller.end(handle, timeout=1))) + closer.start() + resume_event = event_q.get(timeout=1) + closer.join(timeout=1) + assert resume_event.type == "screen" + assert entry_ordinal < resume_event.source_ordinal + assert result[0].resume_frame.source_ordinal == resume_event.source_ordinal + + terminate.set() + reader.join(timeout=1) + assert not reader.is_alive() + + +def test_input_is_dropped_before_structural_observation(tmp_path, monkeypatch) -> None: + controller = _bound_controller(tmp_path) + controller.begin(methods="password_manager", requires_user_presence=True) + terminate = threading.Event() + event_q = OrderedEventJournal() + observations = [] + + class StructuralObserver: + def observe(self, request): + observations.append(request) + return None + + class FakeObserver: + def __init__(self, callback) -> None: + self.callback = callback + + def start(self) -> None: + self.callback( + ObservedMouseButton( + x=10, + y=20, + button="left", + pressed=True, + timestamp=5.0, + ) + ) + terminate.set() + + def check_health(self) -> None: + return + + def stop(self) -> None: + return + + monkeypatch.setattr( + recorder_module, + "create_input_observer", + lambda callback, **_kwargs: FakeObserver(callback), + ) + recorder_module.read_input_events( + event_q, + terminate, + object(), + threading.Event(), + structural_observer=StructuralObserver(), + authentication=controller, + ) + + assert event_q.empty() + assert observations == [] diff --git a/tests/test_control.py b/tests/test_control.py index 7b24bc2..3d0517b 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -24,13 +24,19 @@ from openadapt_capture import control from openadapt_capture import recorder as recorder_module +from openadapt_capture.authentication import ( + AuthenticationHandoff, + FreshFrameProof, +) from openadapt_capture.capture import CaptureSession from openadapt_capture.config import RecordingConfig, config_override from openadapt_capture.control import ( CaptureControlError, CaptureControlUnavailable, RecorderControlServer, + begin_authentication_handoff, discover_recorders, + end_authentication_handoff, status_recording, stop_recording, ) @@ -339,6 +345,105 @@ def stop(_timeout: float) -> dict: server.close() +def test_authenticated_handoff_control_is_retry_safe(tmp_path: Path) -> None: + state = _terminal_payload(tmp_path / "capture", str(uuid.uuid4())) + intervals: dict[str, AuthenticationHandoff] = {} + + def begin(payload: dict, _timeout: float) -> dict: + interval_id = str(uuid.UUID(payload["interval_id"])) + interval = intervals.setdefault( + interval_id, + AuthenticationHandoff( + interval_id=interval_id, + methods=tuple(payload["methods"]), + requires_user_presence=payload["requires_user_presence"], + saved_account_selected=payload["saved_account_selected"], + started_at=1.0, + suppressed_sources=( + "audio", + "browser", + "input", + "screen", + "structural", + "window", + ), + ), + ) + return { + **state, + "authentication_protected": True, + "authentication": {"interval_id": interval.interval_id}, + } + + def end(payload: dict, _timeout: float) -> dict: + interval = intervals[payload["interval_id"]] + if interval.outcome is None: + interval = AuthenticationHandoff.model_validate( + interval.model_copy( + update={ + "ended_at": 3.0, + "outcome": payload["outcome"], + "resume_frame": FreshFrameProof( + timestamp=2.0, + source_ordinal=2, + frame_sha256="a" * 64, + capture_source="desktop-screenshot", + ), + } + ).model_dump(mode="json") + ) + intervals[interval.interval_id] = interval + return { + **state, + "authentication_protected": False, + "authentication": interval.model_dump(mode="json"), + } + + server = RecorderControlServer( + capture_dir=state["capture_dir"], + snapshot=lambda: dict(state), + stop=lambda _timeout: dict(state), + begin_authentication=begin, + end_authentication=end, + session_id=state["session_id"], + runtime_dir=tmp_path / "runtime", + ).start() + try: + interval_id = str(uuid.uuid4()) + first = begin_authentication_handoff( + methods=("password_manager", "mfa"), + requires_user_presence=True, + saved_account_selected=True, + interval_id=interval_id, + session_id=state["session_id"], + runtime_dir=tmp_path / "runtime", + ) + retried = begin_authentication_handoff( + methods=("password_manager", "mfa"), + requires_user_presence=True, + saved_account_selected=True, + interval_id=interval_id, + session_id=state["session_id"], + runtime_dir=tmp_path / "runtime", + ) + assert first == retried + + completed = end_authentication_handoff( + first, + session_id=state["session_id"], + runtime_dir=tmp_path / "runtime", + ) + repeated = end_authentication_handoff( + first, + session_id=state["session_id"], + runtime_dir=tmp_path / "runtime", + ) + assert completed == repeated + assert completed.outcome == "completed" + finally: + server.close() + + def test_control_thread_start_failure_removes_descriptor( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 8ec056812beb7e9b41e48f5818341c3adbac0ee4 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 31 Aug 2026 11:17:30 -0400 Subject: [PATCH 5/5] fix(macos): preserve provider errors on Python 3.10 --- openadapt_capture/window_capture.py | 7 ++++++- tests/test_window_capture.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 4302109..41bdb2c 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -1951,8 +1951,13 @@ def assert_offscreen_target_is_safe() -> None: ) else WindowCaptureError ) + provider_details = "; ".join( + f"{type(provider_failure).__name__}: {provider_failure}" + for provider_failure in failures + ) failure = failure_type( - f"all exact-window capture providers failed for window {window.window_id}" + f"all exact-window capture providers failed for window " + f"{window.window_id}: {provider_details}" ) for provider_failure in failures: try: diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 1267056..dd8822f 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -1734,7 +1734,7 @@ def test_macos_provider_rechecks_minimized_state_before_utility(monkeypatch): window_capture_module._MacOSWindowCaptureProvider().capture(window) assert utility_calls == [] - assert any("minimized" in note for note in exc_info.value.__notes__) + assert "minimized" in str(exc_info.value) def test_macos_provider_refuses_utility_frame_if_window_becomes_minimized( @@ -1770,7 +1770,7 @@ def utility(*_args, **_kwargs): window_capture_module._MacOSWindowCaptureProvider().capture(window) assert utility_calls == ["capture"] - assert any("minimized" in note for note in exc_info.value.__notes__) + assert "minimized" in str(exc_info.value) def test_macos_minimized_state_matches_exact_ax_window_number(monkeypatch):