From 5f6d31b211044551af988a665166bb2b7dd9bb51 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 15:48:55 -0400 Subject: [PATCH 1/6] feat: seal native action geometry epochs --- README.md | 16 + docs/DESIGN.md | 47 +- openadapt_capture/capture.py | 182 ++++++- openadapt_capture/db/__init__.py | 24 + openadapt_capture/db/crud.py | 22 +- openadapt_capture/db/models.py | 10 + openadapt_capture/desktop_capture.py | 21 +- openadapt_capture/events.py | 137 +++++- openadapt_capture/processing.py | 65 ++- openadapt_capture/recorder.py | 679 +++++++++++++++++++++++---- openadapt_capture/terminal.py | 438 +++++++++++++++++ openadapt_capture/window_capture.py | 334 +++++++++++-- tests/test_capture_terminal.py | 129 +++++ tests/test_desktop_capture.py | 6 +- tests/test_window_capture.py | 75 ++- 15 files changed, 2002 insertions(+), 183 deletions(-) create mode 100644 openadapt_capture/terminal.py create mode 100644 tests/test_capture_terminal.py diff --git a/README.md b/README.md index 00a2ef2..296d988 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,13 @@ In this mode: (`CaptureSession.window_capture`), and the window is re-resolved every frame with bounds changes recorded as window events, a bounds timeline converters can use to be exact even when the window moves. +- **Each frame has one ordered geometry identity.** Current window captures + store the frame and its window event under the same source ordinal. An action + stores a later ordinal and names the exact earlier pair it used. The pair + binds the process start identity, display topology, bounds, scale, fixed + viewport, and geometry generation. Recorder shutdown retains one final frame + after input has stopped, so a consumer can select an exact after-action frame + by ordinal instead of by nearest timestamp. - **Window movement and resize are supported.** The first frame fixes the encoded video size. Later source frames scale to fit and letterbox into that viewport. Input uses the exact current bounds and content rectangle. No frame @@ -277,6 +284,15 @@ Note for converters: window-mode coordinates are already in captured-frame pixels (`coordinate_space == "window_pixels"`); do not rescale them by `pixel_ratio`. +After every producer and writer has stopped, `Recorder` verifies the database +and writes `capture-artifact-manifest.json` plus `capture-terminal.json`. The +terminal binds the complete artifact inventory, event counts, final source +ordinal, capture session identity, and completion time. A current native +consumer should use `CaptureSession.load_verified()`. It checks the seal, copies +the exact artifacts into a private snapshot, and opens the copied database in +SQLite immutable read-only mode. A current window capture without this seal is +incomplete and must not be compiled as native geometry evidence. + ## Multiple monitors Full-screen mode records the complete virtual desktop reported by MSS, not diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 686084f..9ee7809 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -42,13 +42,22 @@ One recording has these stages: 2. Resolve the initial native-window or virtual-desktop coordinate scope. 3. Create the per-capture SQLite database and media staging path. 4. Observe native input and screen frames on separate workers. -5. Put all observed events onto one timestamped processing queue. -6. Associate actionable input with the preceding screen observation and - optional structural observation. -7. Stream RGB frames to a separately provisioned FFmpeg process. -8. Close, verify, and atomically promote the MP4. Retain an incomplete partial +5. Reserve each observation in one ordered source journal before any optional + structural lookup. A failed reservation fails the session; later events + cannot pass it. +6. For a native window, enqueue each frame and its window geometry as one + source-ordinal pair. Publish that geometry to input observers only after the + pair enters the journal. +7. Bind each actionable input to the last published frame pair and optional + structural observation. Retain one ordinal-later frame after input stops. +8. Stream RGB frames to a separately provisioned FFmpeg process. +9. Close, verify, and atomically promote the MP4. Retain an incomplete partial file on an encoder failure and never report it as complete media. -9. Post-process raw input into the public action view. +10. Post-process raw input into the public action view. A merged action keeps + its terminal primitive's source binding and refuses mixed geometry epochs. +11. Reconcile committed rows with producer counts, verify the v2 frame/action + relations, inventory every immutable artifact, and write the completion + seal. A worker failure stops the session and propagates through the recording boundary. A frame whose size violates the fixed stream contract is an error. It @@ -91,6 +100,32 @@ window changed size. Input outside the selected window remains out of range. Capture does not clamp it into a valid-looking target coordinate. +Each current window frame carries a process-bound window identity, display +topology digest, and geometry epoch digest. The window and screenshot rows use +the same source ordinal. An action uses a later ordinal and names the exact pair +that supplied its coordinates. Capture refuses process replacement, topology +drift, off-screen state, mixed generations, or a missing pair. + +## Completion and consumer boundary + +A recorder session becomes complete only after all producers and writers have +stopped and the database has passed its integrity and relationship checks. +Capture then writes two create-only files: + +- `capture-artifact-manifest.json` inventories every immutable regular file by + relative path, size, and SHA-256 digest. +- `capture-terminal.json` binds that manifest, the source session identity, + event counts, last source ordinal, and completion interval. + +Both files use canonical JSON and domain-separated digests. Mutable local +control state is not part of the artifact inventory. + +`CaptureSession.load_verified()` checks the complete inventory before it opens +the database. It copies the verified files to a private temporary directory and +opens the copied database with SQLite `mode=ro&immutable=1`. It never migrates +or writes the source capture. A current v2 window session without the terminal +and manifest is incomplete evidence. + ## Native input Capture records these primitive event classes: diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 4c05142..6d600fe 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -28,6 +28,7 @@ ActionEvent as PydanticActionEvent, ) from openadapt_capture.events import ( + CapturedWindowEvent, KeyDownEvent, KeyShortcutEvent, KeyTypeEvent, @@ -36,6 +37,7 @@ MouseMoveEvent, MouseScrollEvent, MouseUpEvent, + WindowCaptureStateV2, ) from openadapt_capture.processing import process_events from openadapt_capture.structural import StructuralObservation @@ -50,6 +52,15 @@ class InvalidCaptureEvent(ValueError): """A stored input event lacks the data required for deterministic replay.""" +@dataclass(frozen=True) +class CapturedFrame: + """Public identity of one retained source frame.""" + + timestamp: float + source_ordinal: int | None + png_sha256: str | None + + def _parse_structural_observation(raw: object) -> StructuralObservation | None: """Load optional structural evidence without breaking legacy recordings.""" @@ -110,10 +121,15 @@ def _convert_action_event(db_event) -> PydanticActionEvent: """ common = { "timestamp": db_event.timestamp, + "source_ordinal": getattr(db_event, "source_ordinal", None), "structural_observation": _parse_structural_observation( getattr(db_event, "structural_observation", None) ), "screenshot_timestamp": getattr(db_event, "screenshot_timestamp", None), + "screenshot_source_ordinal": getattr(db_event, "screenshot_source_ordinal", None), + "window_event_timestamp": getattr(db_event, "window_event_timestamp", None), + "window_event_source_ordinal": getattr(db_event, "window_event_source_ordinal", None), + "window_geometry_generation": getattr(db_event, "window_geometry_generation", None), } if db_event.name == "move": @@ -140,9 +156,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent: button=button, ) else: - raise InvalidCaptureEvent( - "stored 'click' event has no pressed/released state" - ) + raise InvalidCaptureEvent("stored 'click' event has no pressed/released state") elif db_event.name == "scroll": return MouseScrollEvent( **common, @@ -194,15 +208,19 @@ def _parse_element_ref(raw: dict | None) -> SemanticElementRef | None: ) state_raw = raw.get("state", {}) - state = ElementState( - enabled=state_raw.get("enabled", True), - focused=state_raw.get("focused", False), - visible=state_raw.get("visible", True), - checked=state_raw.get("checked"), - selected=state_raw.get("selected"), - expanded=state_raw.get("expanded"), - value=state_raw.get("value"), - ) if isinstance(state_raw, dict) else ElementState() + state = ( + ElementState( + enabled=state_raw.get("enabled", True), + focused=state_raw.get("focused", False), + visible=state_raw.get("visible", True), + checked=state_raw.get("checked"), + selected=state_raw.get("selected"), + expanded=state_raw.get("expanded"), + value=state_raw.get("value"), + ) + if isinstance(state_raw, dict) + else ElementState() + ) return SemanticElementRef( role=raw.get("role") or "", @@ -318,9 +336,7 @@ def _convert_browser_event(db_event) -> "BrowserEvent | None": tab_id=tab_id, previous_url=payload.get("previousUrl", ""), navigation_type=( - NavigationType(nav_type) - if nav_type in valid - else NavigationType.LINK + NavigationType(nav_type) if nav_type in valid else NavigationType.LINK ), ) elif event_type == BrowserEventType.MOUSEMOVE: @@ -348,6 +364,7 @@ def _convert_browser_event(db_event) -> "BrowserEvent | None": ) except Exception as e: import logging + logging.getLogger(__name__).debug("Failed to parse browser event: %s", e) return None @@ -368,6 +385,11 @@ def timestamp(self) -> float: """Unix timestamp of the action.""" return self.event.timestamp + @property + def source_ordinal(self) -> int | None: + """Return the action's terminal primitive source-journal position.""" + return self.event.source_ordinal + @property def type(self) -> str: """Action type (e.g., 'mouse.singleclick', 'key.type').""" @@ -449,6 +471,31 @@ def structural_observation(self) -> StructuralObservation | None: """Accessibility evidence captured at this action, when available.""" return self.event.structural_observation + @property + def screenshot_timestamp(self) -> float | None: + """Exact retained frame timestamp bound to this action.""" + return self.event.screenshot_timestamp + + @property + def screenshot_source_ordinal(self) -> int | None: + """Return the exact source-journal position of the bound frame.""" + return self.event.screenshot_source_ordinal + + @property + def window_event_timestamp(self) -> float | None: + """Exact atomic WindowEvent timestamp bound to this action.""" + return self.event.window_event_timestamp + + @property + def window_event_source_ordinal(self) -> int | None: + """Return the source-journal position of the bound native geometry.""" + return self.event.window_event_source_ordinal + + @property + def window_geometry_generation(self) -> int | None: + """Exact native geometry generation bound to this action.""" + return self.event.window_geometry_generation + @property def screenshot(self) -> "Image" | None: """Get the exact retained screen frame this action is bound to. @@ -497,6 +544,8 @@ def __init__( self.capture_dir = Path(capture_dir) self._session = session self._recording = recording + self._verified_tempdir = None + self._verified_terminal = None @classmethod def load(cls, capture_dir: str | Path) -> "CaptureSession": @@ -543,6 +592,41 @@ def _discard_session() -> None: return cls(capture_dir, session, recording) + @classmethod + def load_verified(cls, capture_dir: str | Path) -> "CaptureSession": + """Verify a completed capture, snapshot it, and open its DB immutable.""" + from openadapt_capture.db import get_immutable_session_for_path + from openadapt_capture.db.models import Recording + from openadapt_capture.terminal import copy_verified_capture + + temporary, snapshot_dir, terminal = copy_verified_capture(capture_dir) + session = get_immutable_session_for_path(str(snapshot_dir / "recording.db")) + + def _discard() -> None: + bind = session.get_bind() + session.close() + if bind is not None: + bind.dispose() + temporary.cleanup() + + try: + recording = session.query(Recording).first() + except Exception: + _discard() + raise + if recording is None: + _discard() + raise FileNotFoundError(f"Invalid capture (no recording found): {capture_dir}") + result = cls(snapshot_dir, session, recording) + result._verified_tempdir = temporary + result._verified_terminal = terminal + return result + + @property + def terminal(self): + """Return the verified immutable terminal, or None for a legacy load.""" + return self._verified_terminal + @property def id(self) -> str: """Capture ID.""" @@ -702,6 +786,71 @@ def raw_events(self) -> list[PydanticActionEvent]: events.append(_convert_action_event(db_event)) return events + def window_events(self) -> list[CapturedWindowEvent]: + """Return stored window rows through the public validated event view.""" + result: list[CapturedWindowEvent] = [] + for row in self._recording.window_events: + state = getattr(row, "state", None) + if not isinstance(state, dict): + raise InvalidCaptureEvent( + f"stored WindowEvent at {row.timestamp!r} has invalid state" + ) + result.append( + CapturedWindowEvent( + timestamp=row.timestamp, + source_ordinal=getattr(row, "source_ordinal", None), + title=row.title, + left=row.left, + top=row.top, + width=row.width, + height=row.height, + window_id=row.window_id, + state=state, + ) + ) + return result + + def frames(self) -> list[CapturedFrame]: + """Return the retained frame timeline without exposing database rows.""" + return [ + CapturedFrame( + timestamp=row.timestamp, + source_ordinal=getattr(row, "source_ordinal", None), + png_sha256=getattr(row, "png_sha256", None), + ) + for row in self._recording.screenshots + ] + + def window_capture_events_v2(self) -> list[CapturedWindowEvent]: + """Return all rows from a declared v2 window-scoped session.""" + metadata = self.window_capture + if metadata is None: + return [] + if metadata.get("schema_version") != "openadapt.capture.window-scoped/v2": + raise InvalidCaptureEvent( + "window-scoped capture does not declare the supported v2 schema" + ) + result = self.window_events() + if not result: + raise InvalidCaptureEvent("v2 window-scoped capture has no window events") + for event in result: + state: WindowCaptureStateV2 | None = event.window_capture_v2 + if state is None: + raise InvalidCaptureEvent("v2 window-scoped capture contains a non-v2 window event") + if event.window_id != state.window_id: + raise InvalidCaptureEvent("stored WindowEvent identity differs from its v2 state") + x, y, width, height = state.bounds + if ( + event.left != int(x) + or event.top != int(y) + or event.width != int(width) + or event.height != int(height) + ): + raise InvalidCaptureEvent("stored WindowEvent columns differ from their v2 bounds") + if not state.on_screen: + raise InvalidCaptureEvent("v2 window event retained an off-screen target") + return result + def actions(self, include_moves: bool = False) -> Iterator[Action]: """Iterate over processed actions. @@ -825,6 +974,9 @@ def close(self) -> None: if bind is not None: bind.dispose() self._session = None + if self._verified_tempdir is not None: + self._verified_tempdir.cleanup() + self._verified_tempdir = None def __enter__(self) -> "CaptureSession": """Context manager entry.""" diff --git a/openadapt_capture/db/__init__.py b/openadapt_capture/db/__init__.py index 96fda3f..5173008 100644 --- a/openadapt_capture/db/__init__.py +++ b/openadapt_capture/db/__init__.py @@ -3,6 +3,9 @@ Copied from legacy OpenAdapt db/db.py, adapted for per-capture databases. """ +import sqlite3 +from pathlib import Path + import sqlalchemy as sa from sqlalchemy import create_engine, inspect, text from sqlalchemy.ext.declarative import declarative_base @@ -166,3 +169,24 @@ def get_session_for_path(db_path: str, echo: bool = False): # a lingering handle makes the capture directory undeletable). engine.dispose() raise + + +def get_immutable_session_for_path(db_path: str, echo: bool = False): + """Open an already-verified SQLite snapshot without schema migration.""" + + resolved = str(Path(db_path).resolve()) + + def _connect() -> sqlite3.Connection: + return sqlite3.connect( + f"file:{resolved}?mode=ro&immutable=1", + uri=True, + check_same_thread=False, + ) + + engine = create_engine("sqlite://", creator=_connect, echo=echo) + Session = get_session_maker(engine) + try: + return Session() + except Exception: + engine.dispose() + raise diff --git a/openadapt_capture/db/crud.py b/openadapt_capture/db/crud.py index 2876a52..6de6b5e 100644 --- a/openadapt_capture/db/crud.py +++ b/openadapt_capture/db/crud.py @@ -343,20 +343,34 @@ def post_process_events(session: SaSession, recording: Recording) -> None: screenshot_timestamp_to_id_map = { screenshot.timestamp: screenshot.id for screenshot in screenshots_list } + screenshot_ordinal_to_id_map = { + screenshot.source_ordinal: screenshot.id + for screenshot in screenshots_list + if screenshot.source_ordinal is not None + } window_event_timestamp_to_id_map = { window_event.timestamp: window_event.id for window_event in window_events_list } + window_event_ordinal_to_id_map = { + window_event.source_ordinal: window_event.id + for window_event in window_events_list + if window_event.source_ordinal is not None + } browser_event_timestamp_to_id_map = { browser_event.timestamp: browser_event.id for browser_event in browser_events_list } for action_event in action_events_list: - action_event.screenshot_id = screenshot_timestamp_to_id_map.get( - action_event.screenshot_timestamp + action_event.screenshot_id = ( + screenshot_ordinal_to_id_map.get(action_event.screenshot_source_ordinal) + if action_event.screenshot_source_ordinal is not None + else screenshot_timestamp_to_id_map.get(action_event.screenshot_timestamp) ) - action_event.window_event_id = window_event_timestamp_to_id_map.get( - action_event.window_event_timestamp + action_event.window_event_id = ( + window_event_ordinal_to_id_map.get(action_event.window_event_source_ordinal) + if action_event.window_event_source_ordinal is not None + else window_event_timestamp_to_id_map.get(action_event.window_event_timestamp) ) action_event.browser_event_id = browser_event_timestamp_to_id_map.get( action_event.browser_event_timestamp diff --git a/openadapt_capture/db/models.py b/openadapt_capture/db/models.py index 59a9dc1..26101b5 100644 --- a/openadapt_capture/db/models.py +++ b/openadapt_capture/db/models.py @@ -95,13 +95,16 @@ class ActionEvent(Base): __tablename__ = "action_event" id = sa.Column(sa.Integer, primary_key=True) + source_ordinal = sa.Column(sa.Integer) name = sa.Column(sa.String) timestamp = sa.Column(ForceFloat) recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) screenshot_timestamp = sa.Column(ForceFloat) + screenshot_source_ordinal = sa.Column(sa.Integer) screenshot_id = sa.Column(sa.ForeignKey("screenshot.id")) window_event_timestamp = sa.Column(ForceFloat) + window_event_source_ordinal = sa.Column(sa.Integer) window_event_id = sa.Column(sa.ForeignKey("window_event.id")) browser_event_timestamp = sa.Column(ForceFloat) browser_event_id = sa.Column(sa.ForeignKey("browser_event.id")) @@ -127,6 +130,9 @@ class ActionEvent(Base): # Versioned optional accessibility evidence captured at action time. # Nullable + additive migration keep older recording.db files readable. structural_observation = sa.Column(sa.JSON) + # Exact native geometry generation bound to the action's retained frame. + # Nullable keeps legacy and full-desktop captures readable. + window_geometry_generation = sa.Column(sa.Integer) disabled = sa.Column(sa.Boolean, default=False) children = sa.orm.relationship("ActionEvent") @@ -167,6 +173,7 @@ class WindowEvent(Base): __tablename__ = "window_event" id = sa.Column(sa.Integer, primary_key=True) + source_ordinal = sa.Column(sa.Integer) recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) timestamp = sa.Column(ForceFloat) @@ -188,6 +195,7 @@ class BrowserEvent(Base): __tablename__ = "browser_event" id = sa.Column(sa.Integer, primary_key=True) + source_ordinal = sa.Column(sa.Integer) recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) message = sa.Column(sa.JSON) @@ -203,10 +211,12 @@ class Screenshot(Base): __tablename__ = "screenshot" id = sa.Column(sa.Integer, primary_key=True) + source_ordinal = sa.Column(sa.Integer) recording_timestamp = sa.Column(ForceFloat) recording_id = sa.Column(sa.ForeignKey("recording.id")) timestamp = sa.Column(ForceFloat) png_data = sa.Column(sa.LargeBinary) + png_sha256 = sa.Column(sa.String) png_diff_data = sa.Column(sa.LargeBinary, nullable=True) png_diff_mask_data = sa.Column(sa.LargeBinary, nullable=True) diff --git a/openadapt_capture/desktop_capture.py b/openadapt_capture/desktop_capture.py index dbfdeec..93f05a4 100644 --- a/openadapt_capture/desktop_capture.py +++ b/openadapt_capture/desktop_capture.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import json import threading import time from dataclasses import dataclass @@ -147,7 +149,7 @@ def translate(self, x: float, y: float) -> tuple[float, float]: def snapshot(self) -> dict[str, Any]: """Return privacy-safe topology metadata retained with the session.""" - return { + payload = { "coordinate_space": "virtual_desktop_pixels", "origin": [self.left, self.top], "viewport": [self.width, self.height], @@ -162,6 +164,23 @@ def snapshot(self) -> dict[str, Any]: for monitor in self.monitors ], } + digest = hashlib.sha256( + json.dumps( + { + "schema_domain": "openadapt.capture.display-topology/v1", + **payload, + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + return { + "schema_version": "openadapt.capture.display-topology/v1", + **payload, + "topology_sha256": digest, + } def _read_current_monitors() -> list[Mapping[str, Any]]: diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 9c1362a..b248198 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -7,12 +7,17 @@ from __future__ import annotations +import math from enum import Enum -from typing import Literal +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from openadapt_capture.structural import StructuralObservation +from openadapt_capture.window_capture import ( + WINDOW_CAPTURE_SCHEMA_VERSION, + window_geometry_epoch_sha256, +) class EventType(str, Enum): @@ -59,11 +64,120 @@ class BaseEvent(BaseModel): """ timestamp: float = Field(description="Unix timestamp in seconds (float for sub-ms precision)") + source_ordinal: int | None = Field( + default=None, + ge=1, + description="Exact 1-based position in the ordered native source journal", + ) type: EventType = Field(description="Event type identifier") model_config = {"use_enum_values": True} +class WindowCaptureStateV2(BaseModel): + """Exact native geometry retained with one atomic screen frame.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.capture.window-scoped/v2"] + window_capture: Literal[True] + window_id: str = Field(min_length=1) + owner: str = Field(min_length=1) + pid: int = Field(gt=0) + process_start_time: float = Field(gt=0) + coordinate_source: str = Field(min_length=1) + geometry_generation: int = Field(ge=1) + geometry_epoch_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + display_topology_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + bounds: tuple[float, float, float, float] + scale: float = Field(gt=0) + scale_x: float = Field(gt=0) + scale_y: float = Field(gt=0) + viewport: tuple[int, int] + source_viewport: tuple[int, int] + content_rect: tuple[int, int, int, int] + fit_scale: float = Field(gt=0) + on_screen: bool + + @model_validator(mode="after") + def _closed_geometry(self) -> "WindowCaptureStateV2": + if self.schema_version != WINDOW_CAPTURE_SCHEMA_VERSION: + raise ValueError("unsupported window-capture schema version") + numeric = ( + self.process_start_time, + *self.bounds, + self.scale, + self.scale_x, + self.scale_y, + self.fit_scale, + ) + if not all(math.isfinite(float(value)) for value in numeric): + raise ValueError("window capture geometry must be finite") + if self.bounds[2] <= 0 or self.bounds[3] <= 0: + raise ValueError("window capture bounds must have positive dimensions") + if any(value <= 0 for value in (*self.viewport, *self.source_viewport)): + raise ValueError("window capture viewports must be positive") + left, top, width, height = self.content_rect + if ( + left < 0 + or top < 0 + or width <= 0 + or height <= 0 + or left + width > self.viewport[0] + or top + height > self.viewport[1] + ): + raise ValueError("window capture content rectangle is outside its viewport") + expected_fit = min( + self.viewport[0] / self.source_viewport[0], + self.viewport[1] / self.source_viewport[1], + ) + expected_width = max( + 1, + min(self.viewport[0], round(self.source_viewport[0] * expected_fit)), + ) + expected_height = max( + 1, + min(self.viewport[1], round(self.source_viewport[1] * expected_fit)), + ) + expected_rect = ( + (self.viewport[0] - expected_width) // 2, + (self.viewport[1] - expected_height) // 2, + expected_width, + expected_height, + ) + if not math.isclose(self.fit_scale, expected_fit) or self.content_rect != expected_rect: + raise ValueError("window capture normalization differs from its viewports") + if not math.isclose(self.scale, self.scale_x): + raise ValueError("legacy window scale differs from exact x scale") + if self.geometry_epoch_sha256 != window_geometry_epoch_sha256( + self.model_dump(mode="json", exclude={"geometry_epoch_sha256"}) + ): + raise ValueError("window geometry epoch digest is invalid") + return self + + +class CapturedWindowEvent(BaseModel): + """Public validated view of one stored WindowEvent row.""" + + model_config = ConfigDict(extra="forbid") + + timestamp: float + source_ordinal: int | None = Field(default=None, ge=1) + title: str | None = None + left: int | None = None + top: int | None = None + width: int | None = None + height: int | None = None + window_id: str | None = None + state: dict[str, Any] + + @property + def window_capture_v2(self) -> WindowCaptureStateV2 | None: + if self.state.get("schema_version") != WINDOW_CAPTURE_SCHEMA_VERSION: + return None + return WindowCaptureStateV2.model_validate(self.state) + + class ActionBaseEvent(BaseEvent): """Base event for native actions with optional structural evidence.""" @@ -79,6 +193,25 @@ class ActionBaseEvent(BaseEvent): "nearest one." ), ) + screenshot_source_ordinal: int | None = Field( + default=None, + ge=1, + description="Source journal ordinal of the exact retained screen frame", + ) + window_event_timestamp: float | None = Field( + default=None, + description="Exact WindowEvent timestamp paired with the bound frame", + ) + window_event_source_ordinal: int | None = Field( + default=None, + ge=1, + description="Source journal ordinal of the geometry paired with the frame", + ) + window_geometry_generation: int | None = Field( + default=None, + ge=1, + description="Exact native geometry generation bound to this action", + ) # ============================================================================= diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index 557aaa4..b302e4f 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -105,6 +105,51 @@ def _bound_screenshot_timestamp(events: list[ActionEvent]) -> float | None: return bound +def _merged_frame_binding(events: list[ActionEvent]) -> dict[str, float | int | None]: + """Return the terminal child binding and reject a mixed native epoch.""" + if not events: + return { + "source_ordinal": None, + "screenshot_timestamp": None, + "screenshot_source_ordinal": None, + "window_event_timestamp": None, + "window_event_source_ordinal": None, + "window_geometry_generation": None, + } + generations = [event.window_geometry_generation for event in events] + if any(value is not None for value in generations): + if any(value is None for value in generations): + raise ValueError( + "cannot merge native action primitives with an incomplete geometry binding" + ) + if len(set(generations)) != 1: + raise ValueError( + "cannot merge native action primitives across geometry epochs" + ) + for event in events: + if ( + event.screenshot_timestamp is None + or event.screenshot_source_ordinal is None + or event.window_event_timestamp is None + or event.window_event_source_ordinal is None + or event.screenshot_timestamp != event.window_event_timestamp + or event.screenshot_source_ordinal + != event.window_event_source_ordinal + ): + raise ValueError( + "cannot merge a native action without one atomic frame/window pair" + ) + terminal = events[-1] + return { + "source_ordinal": terminal.source_ordinal, + "screenshot_timestamp": terminal.screenshot_timestamp, + "screenshot_source_ordinal": terminal.screenshot_source_ordinal, + "window_event_timestamp": terminal.window_event_timestamp, + "window_event_source_ordinal": terminal.window_event_source_ordinal, + "window_geometry_generation": terminal.window_geometry_generation, + } + + # ============================================================================= # Event Processing Functions # ============================================================================= @@ -265,9 +310,7 @@ def flush_buffer() -> None: structural_observation=_first_structural_observation( list(keyboard_buffer) ), - screenshot_timestamp=_bound_screenshot_timestamp( - list(keyboard_buffer) - ), + **_merged_frame_binding(list(keyboard_buffer)), ) ) else: @@ -284,7 +327,7 @@ def flush_buffer() -> None: structural_observation=_first_structural_observation( list(keyboard_buffer) ), - screenshot_timestamp=_bound_screenshot_timestamp(list(keyboard_buffer)), + **_merged_frame_binding(list(keyboard_buffer)), ) result.append(type_event) @@ -347,7 +390,7 @@ def flush_buffer() -> None: structural_observation=_first_structural_observation( list(move_buffer) ), - screenshot_timestamp=_bound_screenshot_timestamp(list(move_buffer)), + **_merged_frame_binding(list(move_buffer)), ) result.append(merged) @@ -400,7 +443,7 @@ def flush_buffer() -> None: structural_observation=_first_structural_observation( list(scroll_buffer) ), - screenshot_timestamp=_bound_screenshot_timestamp(list(scroll_buffer)), + **_merged_frame_binding(list(scroll_buffer)), ) result.append(merged) @@ -508,9 +551,7 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: structural_observation=_first_structural_observation( [down, up, next_down, next_up] ), - screenshot_timestamp=_bound_screenshot_timestamp( - [down, up, next_down, next_up] - ), + **_merged_frame_binding([down, up, next_down, next_up]), ) result.append(double_click) skip_timestamps.add(up.timestamp) @@ -528,7 +569,7 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: structural_observation=_first_structural_observation( [down, up] ), - screenshot_timestamp=_bound_screenshot_timestamp([down, up]), + **_merged_frame_binding([down, up]), ) result.append(single_click) skip_timestamps.add(up.timestamp) @@ -596,9 +637,7 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: structural_observation=_first_structural_observation( [down_event] + moves + [event] ), - screenshot_timestamp=_bound_screenshot_timestamp( - [down_event] + moves + [event] - ), + **_merged_frame_binding([down_event] + moves + [event]), ) result.append(drag) else: diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index ead42c3..e99a01f 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -19,7 +19,9 @@ """ +import hashlib import io +import math import multiprocessing import os import queue @@ -30,7 +32,8 @@ import time import tracemalloc import uuid -from collections import namedtuple +from collections import deque, namedtuple +from dataclasses import dataclass from functools import partial from typing import Any, Callable @@ -63,12 +66,23 @@ observe_structural_action, ) from openadapt_capture.window_capture import ( + WindowCaptureError, WindowCaptureScope, build_window_scope, ) CoordinateScope = WindowCaptureScope | DesktopCaptureScope + +@dataclass(frozen=True) +class WindowScopedFrame: + """One ordered frame plus the exact native geometry that produced it.""" + + image: Any + window_event_data: dict[str, Any] + geometry_generation: int + + try: import soundfile except ImportError: @@ -105,7 +119,164 @@ def _send_profiling_via_wormhole(profile_path: str, timeout: int = 60) -> None: print(f"\nCancelled. File at: {profile_path}") -Event = namedtuple("Event", ("timestamp", "type", "data")) +Event = namedtuple( + "Event", + ("timestamp", "type", "data", "source_ordinal"), + defaults=(None,), +) + + +class EventJournalOrderingError(RuntimeError): + """An event producer violated the ordered source journal.""" + + +class EventJournalReservationError(RuntimeError): + """A producer failed after it reserved a source journal position.""" + + +@dataclass +class _JournalEntry: + timestamp: float + sequence: int + event: Event | None = None + error: BaseException | None = None + ready: bool = False + + +class EventReservation: + """A producer-owned journal position completed after observation work.""" + + def __init__(self, journal: "OrderedEventJournal", entry: _JournalEntry) -> None: + self._journal = journal + self._entry = entry + self._finished = False + + @property + def source_ordinal(self) -> int: + return self._entry.sequence + + def complete(self, event: Event) -> None: + if self._finished: + raise RuntimeError("the event journal reservation is already complete") + if event.timestamp != self._entry.timestamp: + raise EventJournalOrderingError( + "the completed event timestamp differs from its reservation" + ) + if event.source_ordinal not in (None, self._entry.sequence): + raise EventJournalOrderingError( + "the completed event ordinal differs from its reservation" + ) + with self._journal._condition: + self._entry.event = event._replace(source_ordinal=self._entry.sequence) + self._entry.ready = True + self._finished = True + self._journal._condition.notify_all() + + def fail(self, error: BaseException) -> None: + if self._finished: + return + with self._journal._condition: + self._entry.error = error + self._entry.ready = True + self._finished = True + self._journal._condition.notify_all() + + +class OrderedEventJournal: + """A causal FIFO journal with pre-observation reservations.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._entries: deque[_JournalEntry] = deque() + self._last_timestamp: float | None = None + self._next_sequence = 1 + + def reserve(self, timestamp: float) -> EventReservation: + timestamp = float(timestamp) + if not math.isfinite(timestamp): + raise EventJournalOrderingError("event timestamps must be finite") + with self._condition: + if self._last_timestamp is not None and timestamp < self._last_timestamp: + raise EventJournalOrderingError( + "an event arrived behind a newer journal reservation" + ) + entry = _JournalEntry(timestamp, self._next_sequence) + self._next_sequence += 1 + self._last_timestamp = timestamp + self._entries.append(entry) + self._condition.notify_all() + return EventReservation(self, entry) + + def put(self, event: Event, block: bool = True, timeout: float | None = None) -> None: + del block, timeout + reservation = self.reserve(event.timestamp) + reservation.complete(event) + + def commit_window_frame( + self, + event: Event, + window_scope: WindowCaptureScope, + generation: int, + ) -> None: + """Append one frame and publish its geometry in one critical section.""" + timestamp = float(event.timestamp) + if not math.isfinite(timestamp): + raise EventJournalOrderingError("event timestamps must be finite") + failure: BaseException | None = None + with self._condition: + if self._last_timestamp is not None and timestamp < self._last_timestamp: + raise EventJournalOrderingError( + "a frame arrived behind a newer journal reservation" + ) + entry = _JournalEntry(timestamp, self._next_sequence) + self._next_sequence += 1 + self._last_timestamp = timestamp + self._entries.append(entry) + try: + window_scope.publish_frame(generation) + except BaseException as exc: + entry.error = exc + failure = exc + else: + entry.event = event._replace(source_ordinal=entry.sequence) + entry.ready = True + self._condition.notify_all() + if failure is not None: + raise failure + + def get(self, block: bool = True, timeout: float | None = None) -> Event: + deadline = None if timeout is None else time.monotonic() + timeout + with self._condition: + while True: + if self._entries and self._entries[0].ready: + entry = self._entries.popleft() + if entry.error is not None: + raise EventJournalReservationError( + "an ordered event producer failed after reservation" + ) from entry.error + if entry.event is None: + raise EventJournalReservationError( + "an ordered event reservation completed without an event" + ) + return entry.event + if not block: + raise queue.Empty + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + raise queue.Empty + self._condition.wait(remaining) + + def get_nowait(self) -> Event: + return self.get(block=False) + + def empty(self) -> bool: + with self._condition: + return not self._entries + + def qsize(self) -> int: + with self._condition: + return len(self._entries) + EVENT_TYPES = ("screen", "action", "window", "browser") LOG_LEVEL = "INFO" @@ -352,6 +523,7 @@ def process_events( num_window_events: multiprocessing.Value, num_browser_events: multiprocessing.Value, num_video_events: multiprocessing.Value, + producers_finished: threading.Event | None = None, ) -> None: """Process events from the event queue and write them to write queues. @@ -371,6 +543,9 @@ def process_events( num_window_events: A counter for the number of window events. num_browser_events: A counter for the number of browser events. num_video_events: A counter for the number of video events. + producers_finished: Set after every event-journal producer has exited. + When supplied, the processor drains the journal to empty after that + boundary instead of racing the shared stop signal. """ utils.set_start_time(recording.timestamp) @@ -381,8 +556,16 @@ def process_events( prev_window_event = None prev_saved_screen_timestamp = 0 prev_saved_window_timestamp = 0 + prev_saved_screen_ordinal = 0 + prev_saved_window_ordinal = 0 started = False - while not terminate_processing.is_set() or not event_q.empty(): + + def processing_complete() -> bool: + if producers_finished is not None: + return producers_finished.is_set() and event_q.empty() + return terminate_processing.is_set() and event_q.empty() + + while not processing_complete(): # Bounded get: a bare event_q.get() deadlocks shutdown when terminate # is set while the queue is empty and the readers have already exited # (nobody left to feed an event, so the loop condition is never @@ -398,18 +581,46 @@ def process_events( assert event.type in EVENT_TYPES, event if prev_event is not None: try: - assert event.timestamp > prev_event.timestamp, ( - event, - prev_event, - ) + if event.source_ordinal is not None and prev_event.source_ordinal is not None: + assert event.source_ordinal > prev_event.source_ordinal, ( + event, + prev_event, + ) + else: + assert event.timestamp > prev_event.timestamp, ( + event, + prev_event, + ) except AssertionError: delta = event.timestamp - prev_event.timestamp log_prev_event = prev_event._replace(data="") log_event = event._replace(data="") logger.error(f"{delta=} {log_prev_event=} {log_event=}") + if event.source_ordinal is not None: + raise EventJournalOrderingError( + "the stored source journal is not strictly ordered" + ) # behavior undefined, swallow for now # XXX TODO: mitigate if event.type == "screen": + scoped_pair = False + if isinstance(event.data, WindowScopedFrame): + scoped_pair = True + scoped_frame = event.data + metadata_generation = scoped_frame.window_event_data.get("state", {}).get( + "geometry_generation" + ) + if metadata_generation != scoped_frame.geometry_generation: + raise WindowCaptureError( + "the scoped frame geometry generation differs from its metadata" + ) + prev_window_event = Event( + event.timestamp, + "window", + scoped_frame.window_event_data, + event.source_ordinal, + ) + event = event._replace(data=scoped_frame.image) prev_screen_event = event if config.RECORD_FULL_VIDEO: video_event = event._replace(type="screen/video") @@ -421,6 +632,37 @@ def process_events( perf_q, ) num_video_events.value += 1 + if scoped_pair: + process_event( + event, + screen_write_q, + write_screen_event, + recording, + perf_q, + ) + num_screen_events.value += 1 + prev_saved_screen_timestamp = event.timestamp + prev_saved_screen_ordinal = event.source_ordinal or 0 + assert prev_window_event is not None + process_event( + prev_window_event, + window_write_q, + write_window_event, + recording, + perf_q, + ) + num_window_events.value += 1 + prev_saved_window_timestamp = prev_window_event.timestamp + prev_saved_window_ordinal = prev_window_event.source_ordinal or 0 + if config.RECORD_VIDEO and not config.RECORD_FULL_VIDEO: + process_event( + event._replace(type="screen/video"), + video_write_q, + write_video_event, + recording, + perf_q, + ) + num_video_events.value += 1 elif event.type == "window": prev_window_event = event elif event.type == "browser": @@ -439,6 +681,7 @@ def process_events( continue else: event.data["screenshot_timestamp"] = prev_screen_event.timestamp + event.data["screenshot_source_ordinal"] = prev_screen_event.source_ordinal if prev_window_event is None: if config.RECORD_WINDOW_DATA: @@ -447,6 +690,24 @@ def process_events( # Window capture disabled — skip window timestamp requirement else: event.data["window_event_timestamp"] = prev_window_event.timestamp + event.data["window_event_source_ordinal"] = prev_window_event.source_ordinal + action_generation = event.data.get("window_geometry_generation") + if action_generation is not None: + window_generation = prev_window_event.data.get("state", {}).get( + "geometry_generation" + ) + if action_generation != window_generation: + raise WindowCaptureError( + "the action geometry generation differs from its published frame" + ) + if prev_window_event.timestamp != prev_screen_event.timestamp: + raise WindowCaptureError( + "the action frame and native geometry are not one atomic pair" + ) + if prev_window_event.source_ordinal != prev_screen_event.source_ordinal: + raise WindowCaptureError( + "the action frame and native geometry have different source ordinals" + ) process_event( event, @@ -458,7 +719,12 @@ def process_events( num_action_events.value += 1 - if prev_saved_screen_timestamp < prev_screen_event.timestamp: + screen_is_new = ( + prev_screen_event.source_ordinal > prev_saved_screen_ordinal + if prev_screen_event.source_ordinal is not None + else prev_saved_screen_timestamp < prev_screen_event.timestamp + ) + if screen_is_new: process_event( prev_screen_event, screen_write_q, @@ -468,6 +734,7 @@ def process_events( ) num_screen_events.value += 1 prev_saved_screen_timestamp = prev_screen_event.timestamp + prev_saved_screen_ordinal = prev_screen_event.source_ordinal or 0 if config.RECORD_VIDEO and not config.RECORD_FULL_VIDEO: prev_video_event = prev_screen_event._replace(type="screen/video") process_event( @@ -479,7 +746,12 @@ def process_events( ) num_video_events.value += 1 if prev_window_event is not None: - if prev_saved_window_timestamp < prev_window_event.timestamp: + window_is_new = ( + prev_window_event.source_ordinal > prev_saved_window_ordinal + if prev_window_event.source_ordinal is not None + else prev_saved_window_timestamp < prev_window_event.timestamp + ) + if window_is_new: process_event( prev_window_event, window_write_q, @@ -489,6 +761,7 @@ def process_events( ) num_window_events.value += 1 prev_saved_window_timestamp = prev_window_event.timestamp + prev_saved_window_ordinal = prev_window_event.source_ordinal or 0 else: raise Exception(f"unhandled {event.type=}") del prev_event @@ -511,7 +784,12 @@ def write_action_event( perf_q: A queue for collecting performance data. """ assert event.type == "action", event - crud.insert_action_event(db, recording, event.timestamp, event.data) + crud.insert_action_event( + db, + recording, + event.timestamp, + {**event.data, "source_ordinal": event.source_ordinal}, + ) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -535,9 +813,13 @@ def write_screen_event( with io.BytesIO() as output: image.save(output, format="PNG") png_data = output.getvalue() - event_data = {"png_data": png_data} + event_data = { + "png_data": png_data, + "png_sha256": hashlib.sha256(png_data).hexdigest(), + } else: - event_data = {} + event_data = {"png_sha256": None} + event_data["source_ordinal"] = event.source_ordinal crud.insert_screenshot(db, recording, event.timestamp, event_data) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -557,7 +839,12 @@ def write_window_event( perf_q: A queue for collecting performance data. """ assert event.type == "window", event - crud.insert_window_event(db, recording, event.timestamp, event.data) + crud.insert_window_event( + db, + recording, + event.timestamp, + {**event.data, "source_ordinal": event.source_ordinal}, + ) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -576,7 +863,12 @@ def write_browser_event( perf_q: A queue for collecting performance data. """ assert event.type == "browser", event - crud.insert_browser_event(db, recording, event.timestamp, event.data) + crud.insert_browser_event( + db, + recording, + event.timestamp, + {**event.data, "source_ordinal": event.source_ordinal}, + ) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) @@ -808,7 +1100,7 @@ def write_video_event( def trigger_action_event( - event_q: queue.Queue, + event_q: queue.Queue | OrderedEventJournal, action_event_args: dict[str, Any], coordinate_scope: CoordinateScope | None = None, timestamp: float | None = None, @@ -832,43 +1124,57 @@ def trigger_action_event( None """ event_timestamp = utils.get_timestamp() if timestamp is None else timestamp - x = action_event_args.get("mouse_x") - y = action_event_args.get("mouse_y") - observation = observe_structural_action( - structural_observer, - StructuralObservationRequest( - event_timestamp=event_timestamp, - action_name=str(action_event_args.get("name") or "unknown"), - x=x, - y=y, - ), + reservation = ( + event_q.reserve(event_timestamp) if isinstance(event_q, OrderedEventJournal) else None ) - if observation is not None: - action_event_args["structural_observation"] = observation.model_dump( - mode="json", - exclude_none=True, - ) - if x is not None and y is not None: - if config.RECORD_READ_ACTIVE_ELEMENT_STATE: - # element lookup needs GLOBAL coordinates: translate afterwards. - element_state = window.get_active_element_state(x, y) - else: - element_state = {} - action_event_args["element_state"] = element_state - if coordinate_scope is not None: - # The recorder captures and validates its first scoped frame before - # starting input. A translation failure after that boundary means - # evidence is incomplete and must terminate the session. + event_data = dict(action_event_args) + x = event_data.get("mouse_x") + y = event_data.get("mouse_y") + try: + if isinstance(coordinate_scope, WindowCaptureScope): + if x is not None and y is not None: + wx, wy, generation = coordinate_scope.translate_with_generation(x, y) + event_data["mouse_x"] = wx + event_data["mouse_y"] = wy + else: + generation = coordinate_scope.generation_for_action() + event_data["window_geometry_generation"] = generation + elif coordinate_scope is not None and x is not None and y is not None: wx, wy = coordinate_scope.translate(x, y) - action_event_args["mouse_x"] = wx - action_event_args["mouse_y"] = wy - event_q.put( - Event( - event_timestamp, - "action", - action_event_args, + event_data["mouse_x"] = wx + event_data["mouse_y"] = wy + + if x is not None and y is not None: + if config.RECORD_READ_ACTIVE_ELEMENT_STATE: + # Element lookup uses the original global coordinates. + element_state = window.get_active_element_state(x, y) + else: + element_state = {} + event_data["element_state"] = element_state + + observation = observe_structural_action( + structural_observer, + StructuralObservationRequest( + event_timestamp=event_timestamp, + action_name=str(event_data.get("name") or "unknown"), + x=x, + y=y, + ), ) - ) + if observation is not None: + event_data["structural_observation"] = observation.model_dump( + mode="json", + exclude_none=True, + ) + event = Event(event_timestamp, "action", event_data) + if reservation is not None: + reservation.complete(event) + else: + event_q.put(event) + except BaseException as exc: + if reservation is not None: + reservation.fail(exc) + raise def on_move( @@ -986,8 +1292,9 @@ def on_scroll( def handle_key( - event_q: queue.Queue, + event_q: queue.Queue | OrderedEventJournal, key: ObservedKey, + coordinate_scope: CoordinateScope | None = None, structural_observer: StructuralObserver | None = None, ) -> None: """Persist a normalized native key transition. @@ -1010,19 +1317,21 @@ def handle_key( "canonical_key_char": key.canonical_key_char, "canonical_key_vk": key.canonical_key_vk, }, + coordinate_scope=coordinate_scope, timestamp=key.timestamp, structural_observer=structural_observer if key.pressed else None, ) def read_screen_events( - event_q: queue.Queue, + event_q: queue.Queue | OrderedEventJournal, terminate_processing: multiprocessing.Event, recording: Recording, started_event: threading.Event, _screen_timing: _ScreenTimingStats | None = None, window_scope: WindowCaptureScope | None = None, desktop_scope: DesktopCaptureScope | None = None, + input_finished: threading.Event | None = None, ) -> None: """Read screen events and add them to the event queue. @@ -1044,6 +1353,8 @@ def read_screen_events( window_scope: Optional window scope for window-pixel-space capture. desktop_scope: Full-screen virtual-desktop contract. It verifies the monitor topology before and after each captured frame. + input_finished: Input-reader completion boundary. Window capture waits + for it before it records the terminal after-action frame. """ if window_scope is not None and desktop_scope is not None: raise ValueError("screen reader cannot use both window and desktop scopes") @@ -1054,23 +1365,15 @@ def read_screen_events( logger.info(f"Starting (fps={fps}, min_interval={min_interval:.3f}s)") started = False - announced_window = False - while not terminate_processing.is_set(): + + def capture_one() -> tuple[float, float]: + nonlocal started t_start = time.perf_counter() if window_scope is not None: # Any failed capture terminates the session. Retrying would omit a # frame while input continues and could produce complete-looking # evidence with a missing interval. - screenshot, window_changed = window_scope.capture_frame() - if window_changed or not announced_window: - event_q.put( - Event( - utils.get_timestamp(), - "window", - window_scope.window_event_data(), - ) - ) - announced_window = True + screenshot, _window_changed = window_scope.capture_frame(publish=False) elif desktop_scope is not None: # A monitor can move or change scale while the combined frame keeps # the same dimensions. Check both sides of the grab so neither the @@ -1082,12 +1385,31 @@ def read_screen_events( screenshot = utils.take_screenshot() t_screenshot = time.perf_counter() if screenshot is None: - logger.warning("Screenshot was None") - continue + raise WindowCaptureError("the captured screenshot was empty") if not started: started_event.set() started = True - event_q.put(Event(utils.get_timestamp(), "screen", screenshot)) + frame_timestamp = utils.get_timestamp() + if window_scope is not None: + if not isinstance(event_q, OrderedEventJournal): + raise WindowCaptureError("window-scoped capture requires the ordered event journal") + generation = window_scope.current_generation() + scoped_frame = WindowScopedFrame( + image=screenshot, + window_event_data=window_scope.window_event_data(), + geometry_generation=generation, + ) + event_q.commit_window_frame( + Event(frame_timestamp, "screen", scoped_frame), + window_scope, + generation, + ) + else: + event_q.put(Event(frame_timestamp, "screen", screenshot)) + return t_start, t_screenshot + + while not terminate_processing.is_set(): + t_start, t_screenshot = capture_one() # Throttle: sleep for the remainder of the frame interval if min_interval > 0: elapsed = time.perf_counter() - t_start @@ -1097,6 +1419,13 @@ def read_screen_events( if _screen_timing is not None: t_end = time.perf_counter() _screen_timing.append((t_screenshot - t_start, t_end - t_start)) + + if window_scope is not None and input_finished is not None: + input_finished.wait() + t_start, t_screenshot = capture_one() + if _screen_timing is not None: + t_end = time.perf_counter() + _screen_timing.append((t_screenshot - t_start, t_end - t_start)) logger.info("Done") @@ -1336,6 +1665,7 @@ def read_input_events( started_event: threading.Event, coordinate_scope: CoordinateScope | None = None, structural_observer: StructuralObserver | None = None, + finished_event: threading.Event | None = None, ) -> None: """Read globally ordered keyboard and mouse events from one native observer.""" stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] @@ -1382,7 +1712,7 @@ def on_observed(event: ObservedInput) -> None: return logger.debug(f"{event=}") - handle_key(event_q, event, structural_observer) + handle_key(event_q, event, coordinate_scope, structural_observer) if not event.pressed: return @@ -1413,14 +1743,15 @@ def on_observed(event: ObservedInput) -> None: setattr(on_observed, "_openadapt_delivery_thread_stop", stop_hook) utils.set_start_time(recording.timestamp) - observer = create_input_observer( - on_observed, - observe_keyboard=True, - observe_mouse=True, - capture_mouse_moves=True, - ) + observer = None started = False try: + observer = create_input_observer( + on_observed, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + ) observer.start() started = True started_event.set() @@ -1430,8 +1761,10 @@ def on_observed(event: ObservedInput) -> None: terminate_processing.set() raise finally: - if started: + if started and observer is not None: observer.stop() + if finished_event is not None: + finished_event.set() def record_audio( @@ -1679,9 +2012,14 @@ def record( window_title or config.RECORD_WINDOW_TITLE, ) initial_window_frame = None + display_scope = DesktopCaptureScope.current() desktop_scope = None if window_scope is not None: - initial_window_frame, _ = window_scope.capture_frame() + window_scope.bind_display_topology( + display_scope.snapshot(), + display_scope.assert_current, + ) + initial_window_frame, _ = window_scope.capture_frame(publish=False) logger.info( f"window-scoped capture resolved: {window_scope.snapshot()} " f"initial frame {initial_window_frame.size}" @@ -1691,7 +2029,7 @@ def record( # ``utils.take_screenshot``. Retain its origin and translate native # input into that same pixel space so secondary monitors with negative # global coordinates remain aligned with the video. - desktop_scope = DesktopCaptureScope.current() + desktop_scope = display_scope logger.info(f"virtual desktop capture resolved: {desktop_scope.snapshot()}") if structural_observer is None: @@ -1709,7 +2047,29 @@ def record( ) recording_timestamp = recording.timestamp - event_q = queue.Queue() + event_q = OrderedEventJournal() + producers_finished = threading.Event() + input_finished = threading.Event() + if window_scope is not None: + # The preflight frame sizes the fixed stream. Capture again after the + # recording clock starts, then publish pixels and geometry atomically + # before any input observer can bind an action to the epoch. + initial_window_frame, _ = window_scope.capture_frame(publish=False) + initial_generation = window_scope.current_generation() + initial_timestamp = utils.get_timestamp() + event_q.commit_window_frame( + Event( + initial_timestamp, + "screen", + WindowScopedFrame( + image=initial_window_frame, + window_event_data=window_scope.window_event_data(), + geometry_generation=initial_generation, + ), + ), + window_scope, + initial_generation, + ) screen_write_q = sq.SynchronizedQueue() action_write_q = sq.SynchronizedQueue() window_write_q = sq.SynchronizedQueue() @@ -1762,6 +2122,7 @@ def record( _screen_timing, window_scope, desktop_scope, + input_finished, ), terminate_processing, task_errors, @@ -1777,6 +2138,7 @@ def record( task_started_events.setdefault("input_event_reader", threading.Event()), window_scope or desktop_scope, structural_observer, + input_finished, ) input_event_reader = threading.Thread( target=_run_task_fail_loud, @@ -1819,6 +2181,7 @@ def record( num_window_events, num_browser_events, num_video_events, + producers_finished, ) event_processor = threading.Thread( target=_run_task_fail_loud, @@ -2002,14 +2365,22 @@ def record( task_by_name, [ "window_event_reader", - "screen_event_reader", "input_event_reader", - "event_processor", + "screen_event_reader", "audio_recorder", ], timeout=pre_ready_timeout, ) + # The processor can now drain every completed reservation. No producer can + # append a later event after it observes an empty journal. + producers_finished.set() + _join_tasks( + task_by_name, + ["event_processor"], + timeout=pre_ready_timeout, + ) + # No writer can stop while the event processor can still enqueue work. # Signal writer completion only after all producers have exited. terminate_writers.set() @@ -2039,7 +2410,9 @@ def record( add_exception_note(task_error, f"recording task {task_name!r} failed") raise task_error _raise_for_failed_processes(task_by_name) - if desktop_scope is not None: + if window_scope is not None: + window_scope.assert_current() + elif desktop_scope is not None: # Close the interval between the last captured frame and operator stop. # A topology change in that interval still invalidates the session. desktop_scope.assert_current(force=True) @@ -2381,9 +2754,7 @@ def _verify_completed_capture(self) -> None: raise RuntimeError("The finalized Capture database has broken relationships.") recordings = database.execute("SELECT id FROM recording").fetchall() if len(recordings) != 1: - raise RuntimeError( - "The finalized Capture database does not contain one session." - ) + raise RuntimeError("The finalized Capture database does not contain one session.") recording_id = recordings[0][0] expected_counts = { "action_event": self._num_action_events.value, @@ -2401,8 +2772,7 @@ def _verify_completed_capture(self) -> None: f"(expected {expected}, committed {committed})." ) wrong_recording = database.execute( - f"SELECT COUNT(*) FROM {table} " - "WHERE recording_id IS NULL OR recording_id != ?", + f"SELECT COUNT(*) FROM {table} WHERE recording_id IS NULL OR recording_id != ?", (recording_id,), ).fetchone()[0] if wrong_recording: @@ -2429,9 +2799,142 @@ def _verify_completed_capture(self) -> None: raise RuntimeError( "The finalized Capture database has an invalid browser event." ) + metadata = capture.window_capture + if ( + isinstance(metadata, dict) + and metadata.get("schema_version") == "openadapt.capture.window-scoped/v2" + ): + window_events = capture.window_capture_events_v2() + windows_by_ordinal = {event.source_ordinal: event for event in window_events} + screenshots_by_ordinal = { + row.source_ordinal: row for row in capture._recording.screenshots + } + if None in windows_by_ordinal or None in screenshots_by_ordinal: + raise RuntimeError("The finalized v2 capture has an unsequenced frame pair.") + if set(windows_by_ordinal) != set(screenshots_by_ordinal): + raise RuntimeError( + "The finalized v2 capture has an incomplete frame/window pair." + ) + if len(windows_by_ordinal) != len(window_events): + raise RuntimeError("The finalized v2 capture reuses a window source ordinal.") + identity = None + topology_sha256 = None + generation_to_epoch: dict[int, str] = {} + previous_generation = 0 + for ordinal in sorted(windows_by_ordinal): + window_event = windows_by_ordinal[ordinal] + window_state = window_event.window_capture_v2 + assert window_state is not None + screenshot = screenshots_by_ordinal[ordinal] + if screenshot.timestamp != window_event.timestamp: + raise RuntimeError("The finalized v2 frame and geometry timestamps differ.") + current_identity = ( + window_state.window_id, + window_state.pid, + window_state.process_start_time, + window_state.owner.casefold(), + ) + if identity is None: + identity = current_identity + topology_sha256 = window_state.display_topology_sha256 + elif current_identity != identity: + raise RuntimeError( + "The finalized v2 capture changed process-bound window identity." + ) + if window_state.display_topology_sha256 != topology_sha256: + raise RuntimeError("The finalized v2 capture changed display topology.") + generation = window_state.geometry_generation + known_epoch = generation_to_epoch.setdefault( + generation, window_state.geometry_epoch_sha256 + ) + if known_epoch != window_state.geometry_epoch_sha256: + raise RuntimeError("A v2 geometry generation names two different epochs.") + if generation < previous_generation: + raise RuntimeError("The finalized v2 geometry generations move backwards.") + previous_generation = generation + action_ordinals: set[int] = set() + for action in capture._recording.action_events: + if ( + action.source_ordinal is None + or action.screenshot_source_ordinal is None + or action.window_event_source_ordinal is None + or action.window_geometry_generation is None + ): + raise RuntimeError( + "The finalized v2 capture has an incomplete action binding." + ) + if action.source_ordinal in action_ordinals: + raise RuntimeError( + "The finalized v2 capture reuses an action source ordinal." + ) + action_ordinals.add(action.source_ordinal) + if ( + action.screenshot_source_ordinal != action.window_event_source_ordinal + or action.source_ordinal <= action.screenshot_source_ordinal + ): + raise RuntimeError( + "The finalized v2 action does not bind one earlier frame pair." + ) + bound_window = windows_by_ordinal.get(action.window_event_source_ordinal) + bound_screen = screenshots_by_ordinal.get(action.screenshot_source_ordinal) + if bound_window is None or bound_screen is None: + raise RuntimeError("The finalized v2 action names a missing frame pair.") + bound_state = bound_window.window_capture_v2 + assert bound_state is not None + if ( + bound_state.geometry_generation != action.window_geometry_generation + or action.screenshot_id != bound_screen.id + or action.window_event_id + != next( + row.id + for row in capture._recording.window_events + if row.source_ordinal == action.window_event_source_ordinal + ) + ): + raise RuntimeError( + "The finalized v2 action relationship differs from its epoch." + ) + list(capture.actions(include_moves=True)) finally: capture.close() + def _seal_completed_capture(self) -> None: + """Write the immutable terminal after database and writer verification.""" + from pathlib import Path + + from openadapt_capture.terminal import seal_capture + + db_path = Path(self.capture_dir) / "recording.db" + database = sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True) + try: + last_source_ordinal = max( + (database.execute(f"SELECT MAX(source_ordinal) FROM {table}").fetchone()[0] or 0) + for table in ( + "action_event", + "screenshot", + "window_event", + "browser_event", + ) + ) + capture_started_at = database.execute("SELECT timestamp FROM recording").fetchone()[0] + finally: + database.close() + seal_capture( + self.capture_dir, + session_id=self._control_session_id, + process_started_at=self._process_started_at, + capture_started_at=capture_started_at, + capture_ended_at=time.time(), + event_counts={ + "action": self._num_action_events.value, + "screen": self._num_screen_events.value, + "window": self._num_window_events.value, + "browser": self._num_browser_events.value, + "video": self._num_video_events.value, + }, + last_source_ordinal=last_source_ordinal or None, + ) + def _start_control_server(self) -> None: from pathlib import Path @@ -2451,10 +2954,7 @@ def _start_control_server(self) -> None: def _control_stop(self, timeout: float) -> dict[str, Any]: """Idempotently request stop and wait for the one finalization result.""" with self._control_stop_lock: - if ( - not self._finalized_event.is_set() - and not self._terminate_processing.is_set() - ): + if not self._finalized_event.is_set() and not self._terminate_processing.is_set(): try: self._transition_control("stopping") except BaseException as exc: @@ -2528,6 +3028,7 @@ def _run_record(self) -> None: self.check_health() if self._ready_event.is_set(): self._verify_completed_capture() + self._seal_completed_capture() self._transition_control( "complete", complete=True, diff --git a/openadapt_capture/terminal.py b/openadapt_capture/terminal.py new file mode 100644 index 0000000..d655523 --- /dev/null +++ b/openadapt_capture/terminal.py @@ -0,0 +1,438 @@ +"""Immutable completion seal for a stopped native capture.""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +ARTIFACT_MANIFEST_FILENAME = "capture-artifact-manifest.json" +CAPTURE_TERMINAL_FILENAME = "capture-terminal.json" +ARTIFACT_MANIFEST_SCHEMA_VERSION = "openadapt.capture-artifact-manifest/v1" +CAPTURE_TERMINAL_SCHEMA_VERSION = "openadapt.capture-terminal/v2" +_MANIFEST_DOMAIN = b"openadapt.capture-artifact-manifest.v1\0" +_TERMINAL_DOMAIN = b"openadapt.capture-terminal.v2\0" +_EXCLUDED_ARTIFACTS = { + "capture-state.json", + ARTIFACT_MANIFEST_FILENAME, + CAPTURE_TERMINAL_FILENAME, +} + + +class CaptureSealError(RuntimeError): + """A capture seal or one of its inventoried artifacts is invalid.""" + + +class ArtifactRecord(BaseModel): + """One exact regular file in a stopped capture.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + path: str = Field(min_length=1) + size_bytes: int = Field(ge=0) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def _safe_relative_path(self) -> "ArtifactRecord": + path = PurePosixPath(self.path) + if ( + path.is_absolute() + or not path.parts + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError("artifact paths must be safe POSIX-relative paths") + if path.as_posix() in _EXCLUDED_ARTIFACTS: + raise ValueError("mutable or seal metadata cannot inventory itself") + return self + + +class CaptureArtifactManifest(BaseModel): + """Canonical inventory of every stopped capture artifact.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["openadapt.capture-artifact-manifest/v1"] + artifacts: tuple[ArtifactRecord, ...] + + @model_validator(mode="after") + def _ordered_unique_paths(self) -> "CaptureArtifactManifest": + paths = [artifact.path for artifact in self.artifacts] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise ValueError("artifact paths must be unique and sorted") + if "recording.db" not in paths: + raise ValueError("the artifact manifest must inventory recording.db") + return self + + +class CaptureEventCounts(BaseModel): + """Committed event counts at the completion boundary.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + action: int = Field(ge=0) + screen: int = Field(ge=0) + window: int = Field(ge=0) + browser: int = Field(ge=0) + video: int = Field(ge=0) + + +class CaptureTerminal(BaseModel): + """Strict immutable proof that the recorder completed and sealed output.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["openadapt.capture-terminal/v2"] + state: Literal["COMPLETE"] + reason_code: Literal["normal_stop"] + source_capture_session_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + started_at: str = Field(min_length=20) + ended_at: str = Field(min_length=20) + event_counts: CaptureEventCounts + last_source_ordinal: int | None = Field(default=None, ge=1) + artifact_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + artifact_manifest_size_bytes: int = Field(gt=0) + terminal_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def _valid_terminal_digest(self) -> "CaptureTerminal": + payload = self.model_dump(mode="json", exclude={"terminal_sha256"}) + if self.terminal_sha256 != _terminal_sha256(payload): + raise ValueError("capture terminal digest is invalid") + return self + + +def _canonical_json_bytes(payload: object, *, newline: bool) -> bytes: + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return encoded + (b"\n" if newline else b"") + + +def _manifest_sha256(raw_manifest: bytes) -> str: + return hashlib.sha256(_MANIFEST_DOMAIN + raw_manifest).hexdigest() + + +def _terminal_sha256(payload: object) -> str: + return hashlib.sha256( + _TERMINAL_DOMAIN + _canonical_json_bytes(payload, newline=False) + ).hexdigest() + + +def _utc_timestamp(timestamp: float) -> str: + return ( + datetime.fromtimestamp(timestamp, tz=timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def source_capture_session_sha256( + *, session_id: str, process_started_at: float, capture_started_at: float +) -> str: + """Derive a privacy-safe identity for one recorder process session.""" + payload = { + "capture_started_at": capture_started_at, + "process_started_at": process_started_at, + "session_id": session_id, + } + return hashlib.sha256( + b"openadapt.capture-source-session.v1\0" + _canonical_json_bytes(payload, newline=False) + ).hexdigest() + + +def _safe_artifact_path(capture_dir: Path, relative_path: str) -> Path: + record = ArtifactRecord(path=relative_path, size_bytes=0, sha256="0" * 64) + candidate = capture_dir.joinpath(*PurePosixPath(record.path).parts) + try: + candidate.relative_to(capture_dir) + candidate.resolve(strict=True).relative_to(capture_dir) + except (OSError, ValueError) as exc: + raise CaptureSealError("an artifact path escapes the capture directory") from exc + return candidate + + +def _open_stable_regular_file(path: Path) -> tuple[int, os.stat_result]: + before = path.lstat() + if not stat.S_ISREG(before.st_mode): + raise CaptureSealError(f"capture artifact is not a regular file: {path.name}") + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(path, flags) + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or ( + opened.st_dev, + opened.st_ino, + ) != (before.st_dev, before.st_ino): + os.close(fd) + raise CaptureSealError(f"capture artifact changed before reading: {path.name}") + return fd, before + + +def _assert_stable_file(path: Path, before: os.stat_result, after: os.stat_result) -> None: + current = path.lstat() + identity = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) + if identity != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) or identity != ( + current.st_dev, + current.st_ino, + current.st_size, + current.st_mtime_ns, + ): + raise CaptureSealError(f"capture artifact changed while reading: {path.name}") + + +def _hash_regular_file(path: Path) -> tuple[int, str]: + """Hash one stable regular file without following a symbolic link.""" + fd, before = _open_stable_regular_file(path) + try: + opened = os.fstat(fd) + if (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino): + raise CaptureSealError(f"capture artifact changed before hashing: {path.name}") + digest = hashlib.sha256() + size = 0 + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + after = os.fstat(fd) + finally: + os.close(fd) + _assert_stable_file(path, before, after) + if size != before.st_size: + raise CaptureSealError(f"capture artifact size changed while hashing: {path.name}") + return size, digest.hexdigest() + + +def _read_regular_file(path: Path) -> bytes: + """Read one stable regular file through the descriptor that was verified.""" + fd, before = _open_stable_regular_file(path) + chunks: list[bytes] = [] + size = 0 + try: + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + chunks.append(chunk) + size += len(chunk) + after = os.fstat(fd) + finally: + os.close(fd) + _assert_stable_file(path, before, after) + if size != before.st_size: + raise CaptureSealError(f"capture artifact size changed while reading: {path.name}") + return b"".join(chunks) + + +def _copy_verified_regular_file( + source: Path, + destination: Path, + expected: ArtifactRecord, +) -> None: + """Copy one exact source file without following a replacement symlink.""" + source_fd, before = _open_stable_regular_file(source) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) + try: + destination_fd = os.open(destination, flags, 0o600) + except BaseException: + os.close(source_fd) + raise + digest = hashlib.sha256() + size = 0 + try: + try: + while True: + chunk = os.read(source_fd, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + view = memoryview(chunk) + while view: + written = os.write(destination_fd, view) + view = view[written:] + os.fsync(destination_fd) + after = os.fstat(source_fd) + finally: + os.close(destination_fd) + os.close(source_fd) + except BaseException: + destination.unlink(missing_ok=True) + raise + _assert_stable_file(source, before, after) + if (size, digest.hexdigest()) != (expected.size_bytes, expected.sha256): + destination.unlink(missing_ok=True) + raise CaptureSealError(f"capture artifact changed during snapshot: {expected.path}") + + +def build_artifact_manifest(capture_dir: str | os.PathLike[str]) -> CaptureArtifactManifest: + """Inventory all stopped regular files except mutable and seal metadata.""" + root = Path(capture_dir).resolve() + artifacts: list[ArtifactRecord] = [] + for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + relative = path.relative_to(root).as_posix() + if relative in _EXCLUDED_ARTIFACTS: + continue + details = path.lstat() + if stat.S_ISDIR(details.st_mode): + continue + if not stat.S_ISREG(details.st_mode): + raise CaptureSealError(f"capture artifact is not a regular file: {relative}") + size, digest = _hash_regular_file(path) + artifacts.append(ArtifactRecord(path=relative, size_bytes=size, sha256=digest)) + return CaptureArtifactManifest( + schema_version=ARTIFACT_MANIFEST_SCHEMA_VERSION, + artifacts=tuple(artifacts), + ) + + +def _write_new_atomic(path: Path, data: bytes) -> None: + if path.exists() or path.is_symlink(): + raise CaptureSealError(f"refusing to replace existing capture seal: {path.name}") + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) + fd = os.open(temporary, flags, 0o600) + try: + view = memoryview(data) + while view: + written = os.write(fd, view) + view = view[written:] + os.fsync(fd) + finally: + os.close(fd) + try: + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def seal_capture( + capture_dir: str | os.PathLike[str], + *, + session_id: str, + process_started_at: float, + capture_started_at: float, + capture_ended_at: float, + event_counts: dict[str, int], + last_source_ordinal: int | None, +) -> CaptureTerminal: + """Write the manifest and terminal after all capture writers have stopped.""" + root = Path(capture_dir).resolve() + manifest = build_artifact_manifest(root) + manifest_raw = _canonical_json_bytes(manifest.model_dump(mode="json"), newline=True) + manifest_digest = _manifest_sha256(manifest_raw) + _write_new_atomic(root / ARTIFACT_MANIFEST_FILENAME, manifest_raw) + payload = { + "schema_version": CAPTURE_TERMINAL_SCHEMA_VERSION, + "state": "COMPLETE", + "reason_code": "normal_stop", + "source_capture_session_sha256": source_capture_session_sha256( + session_id=session_id, + process_started_at=process_started_at, + capture_started_at=capture_started_at, + ), + "started_at": _utc_timestamp(capture_started_at), + "ended_at": _utc_timestamp(capture_ended_at), + "event_counts": event_counts, + "last_source_ordinal": last_source_ordinal, + "artifact_manifest_sha256": manifest_digest, + "artifact_manifest_size_bytes": len(manifest_raw), + } + payload["terminal_sha256"] = _terminal_sha256(payload) + terminal = CaptureTerminal.model_validate(payload) + terminal_raw = _canonical_json_bytes(terminal.model_dump(mode="json"), newline=True) + _write_new_atomic(root / CAPTURE_TERMINAL_FILENAME, terminal_raw) + return terminal + + +def verify_capture_artifacts( + capture_dir: str | os.PathLike[str], +) -> tuple[CaptureTerminal, CaptureArtifactManifest]: + """Verify canonical seal bytes and every inventoried artifact.""" + root = Path(capture_dir).resolve() + terminal_path = root / CAPTURE_TERMINAL_FILENAME + manifest_path = root / ARTIFACT_MANIFEST_FILENAME + terminal_raw = _read_regular_file(terminal_path) + manifest_raw = _read_regular_file(manifest_path) + try: + terminal = CaptureTerminal.model_validate_json(terminal_raw) + manifest = CaptureArtifactManifest.model_validate_json(manifest_raw) + except Exception as exc: + raise CaptureSealError("capture seal is malformed") from exc + if terminal_raw != _canonical_json_bytes(terminal.model_dump(mode="json"), newline=True): + raise CaptureSealError("capture terminal bytes are not canonical") + if manifest_raw != _canonical_json_bytes(manifest.model_dump(mode="json"), newline=True): + raise CaptureSealError("artifact manifest bytes are not canonical") + if terminal.artifact_manifest_size_bytes != len(manifest_raw): + raise CaptureSealError("artifact manifest size differs from the terminal") + if terminal.artifact_manifest_sha256 != _manifest_sha256(manifest_raw): + raise CaptureSealError("artifact manifest digest differs from the terminal") + expected_paths = {artifact.path for artifact in manifest.artifacts} + actual_paths: set[str] = set() + for path in root.rglob("*"): + relative = path.relative_to(root).as_posix() + if relative in _EXCLUDED_ARTIFACTS: + continue + details = path.lstat() + if stat.S_ISDIR(details.st_mode): + continue + if not stat.S_ISREG(details.st_mode): + raise CaptureSealError(f"capture artifact is not a regular file: {relative}") + actual_paths.add(relative) + if actual_paths != expected_paths: + raise CaptureSealError("capture artifacts differ from the sealed inventory") + for artifact in manifest.artifacts: + size, digest = _hash_regular_file(_safe_artifact_path(root, artifact.path)) + if (size, digest) != (artifact.size_bytes, artifact.sha256): + raise CaptureSealError(f"capture artifact differs from its seal: {artifact.path}") + return terminal, manifest + + +def copy_verified_capture( + capture_dir: str | os.PathLike[str], +) -> tuple[tempfile.TemporaryDirectory[str], Path, CaptureTerminal]: + """Copy a verified capture into a private, stable consumer snapshot.""" + terminal, manifest = verify_capture_artifacts(capture_dir) + source = Path(capture_dir).resolve() + temporary = tempfile.TemporaryDirectory(prefix="openadapt-capture-verified-") + destination = Path(temporary.name) + try: + for artifact in manifest.artifacts: + source_path = _safe_artifact_path(source, artifact.path) + target_path = destination.joinpath(*PurePosixPath(artifact.path).parts) + target_path.parent.mkdir(parents=True, exist_ok=True) + _copy_verified_regular_file(source_path, target_path, artifact) + size, digest = _hash_regular_file(target_path) + if (size, digest) != (artifact.size_bytes, artifact.sha256): + raise CaptureSealError(f"capture artifact changed during snapshot: {artifact.path}") + _write_new_atomic( + destination / ARTIFACT_MANIFEST_FILENAME, + _canonical_json_bytes(manifest.model_dump(mode="json"), newline=True), + ) + _write_new_atomic( + destination / CAPTURE_TERMINAL_FILENAME, + _canonical_json_bytes(terminal.model_dump(mode="json"), newline=True), + ) + except BaseException: + temporary.cleanup() + raise + return temporary, destination, terminal diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index a4c5f1e..5760a96 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -33,6 +33,9 @@ from __future__ import annotations +import hashlib +import json +import math import sys import threading from dataclasses import dataclass @@ -43,6 +46,44 @@ if TYPE_CHECKING: from PIL import Image +WINDOW_CAPTURE_SCHEMA_VERSION = "openadapt.capture.window-scoped/v2" + + +def window_geometry_epoch_sha256(state: dict) -> str: + """Hash the exact native coordinate contract for one published frame.""" + payload = { + key: state.get(key) + for key in ( + "schema_version", + "window_id", + "owner", + "pid", + "process_start_time", + "coordinate_source", + "geometry_generation", + "display_topology_sha256", + "bounds", + "scale", + "scale_x", + "scale_y", + "viewport", + "source_viewport", + "content_rect", + "fit_scale", + ) + } + encoded = json.dumps( + { + "schema_domain": "openadapt.capture.window-geometry-epoch/v1", + **payload, + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + class WindowCaptureError(RuntimeError): """The target window could not be resolved or captured. @@ -109,6 +150,18 @@ class TargetWindow: pid: int bounds: tuple[float, float, float, float] on_screen: bool = True + process_start_time: float | None = None + coordinate_source: str = "platform-screen" + + @property + def identity(self) -> tuple[int, int, float | None, str]: + """Return the process-bound identity used for the whole session.""" + return ( + self.window_id, + self.pid, + self.process_start_time, + self.owner.casefold(), + ) def translate_point( @@ -167,13 +220,45 @@ def __init__( self._source_viewport: tuple[int, int] | None = None self._content_rect: tuple[int, int, int, int] | None = None self._fit_scale: float | None = None - self._bound_window_id: int | None = None + self._geometry_generation = 0 + self._geometry_signature: tuple | None = None + self._published_generation = 0 + self._published_window: TargetWindow | None = None + self._published_scale_x: float | None = None + self._published_scale_y: float | None = None + self._published_content_rect: tuple[int, int, int, int] | None = None + self._bound_identity: tuple[int, int, float | None, str] | None = None + self._display_topology: dict | None = None + self._display_topology_guard: Callable[..., None] | None = None # Window of the last CAPTURED frame (not merely resolved): the # bounds-timeline 'changed' flag compares frame to frame, so a bare # resolve() (e.g. a pre-flight existence check) never suppresses the # first frame's timeline entry. self._frame_window: TargetWindow | None = None + def bind_display_topology( + self, + snapshot: dict, + guard: Callable[..., None], + ) -> None: + """Bind the exact active-display inventory for this recording.""" + if not isinstance(snapshot, dict) or not snapshot.get("topology_sha256"): + raise WindowCaptureError("window capture requires hashed display topology") + with self._lock: + if self._display_topology is not None: + raise WindowCaptureError("display topology is already bound") + self._display_topology = dict(snapshot) + self._display_topology_guard = guard + + def _assert_display_topology(self) -> None: + with self._lock: + guard = self._display_topology_guard + if guard is None: + raise WindowCaptureError( + "window capture requires a bound display-topology guard" + ) + guard(force=True) + def resolve(self) -> TargetWindow: """Resolve the target window without changing captured-frame geometry. @@ -187,9 +272,39 @@ def resolve(self) -> TargetWindow: f"title {self.target.title!r}; is the target application " "running with a visible window?" ) + if self.target.owner and self.target.owner.casefold() not in win.owner.casefold(): + raise WindowCaptureError( + "the window resolver returned an owner outside the configured selector" + ) + if self.target.title and self.target.title.casefold() not in win.title.casefold(): + raise WindowCaptureError( + "the window resolver returned a title outside the configured selector" + ) + if win.pid <= 0: + raise WindowCaptureError("the resolved target has no owning process identity") + if ( + win.process_start_time is None + or not math.isfinite(win.process_start_time) + or win.process_start_time <= 0 + ): + raise WindowCaptureError( + "the resolved target has no stable process start identity" + ) + if not win.coordinate_source.strip(): + raise WindowCaptureError("the resolved target has no coordinate source") return win - def capture_frame(self) -> tuple["Image.Image", bool]: + def _assert_bound_identity(self, win: TargetWindow) -> None: + """Reject a recycled window handle or a different owning process.""" + with self._lock: + bound_identity = self._bound_identity + if bound_identity is not None and win.identity != bound_identity: + raise WindowCaptureError( + "the resolved target changed window identity or owning process " + "during recording" + ) + + def capture_frame(self, *, publish: bool = True) -> tuple["Image.Image", bool]: """Capture the target window's current pixels. Re-resolves the window first so bounds/scale can never disagree with @@ -210,18 +325,29 @@ def capture_frame(self) -> tuple["Image.Image", bool]: """ with self._lock: prev = self._frame_window - bound_window_id = self._bound_window_id - win = self.resolve() - if bound_window_id is not None and win.window_id != bound_window_id: + output_viewport = self._viewport + + self._assert_display_topology() + pre = self.resolve() + self._assert_bound_identity(pre) + source_image = self._capturer(pre) + post = self.resolve() + self._assert_bound_identity(post) + self._assert_display_topology() + if pre.identity != post.identity: + raise WindowCaptureError( + "the target process identity changed while a frame was captured" + ) + if pre.bounds != post.bounds: raise WindowCaptureError( - "the resolved target changed window identity during recording: " - f"expected {bound_window_id}, got {win.window_id}" + "the target moved or resized while a frame was captured; " + "no action can bind to mixed frame geometry" ) - source_image = self._capturer(win) + win = post if source_image.width <= 0 or source_image.height <= 0: raise WindowCaptureError("window capture returned an empty frame") source_viewport = (source_image.width, source_image.height) - output_viewport = self._viewport or source_viewport + output_viewport = output_viewport or source_viewport output_width, output_height = output_viewport fit_scale = min( output_width / source_image.width, @@ -243,6 +369,22 @@ def capture_frame(self) -> tuple["Image.Image", bool]: bounds_h = win.bounds[3] or float(source_image.height) scale_x = fitted_width / bounds_w scale_y = fitted_height / bounds_h + with self._lock: + topology = self._display_topology + topology_sha256 = ( + str(topology["topology_sha256"]) if topology is not None else None + ) + geometry_signature = ( + win.identity, + win.bounds, + win.coordinate_source, + output_viewport, + source_viewport, + (offset_x, offset_y, fitted_width, fitted_height), + scale_x, + scale_y, + topology_sha256, + ) with self._lock: self._window = win # ``scale`` is the historical scalar field. Keep it as the x-axis @@ -255,8 +397,15 @@ def capture_frame(self) -> tuple["Image.Image", bool]: self._source_viewport = source_viewport self._content_rect = (offset_x, offset_y, fitted_width, fitted_height) self._fit_scale = fit_scale - self._bound_window_id = win.window_id + if self._bound_identity is None: + self._bound_identity = win.identity + if geometry_signature != self._geometry_signature: + self._geometry_generation += 1 + self._geometry_signature = geometry_signature self._frame_window = win + generation = self._geometry_generation + if publish: + self._publish_locked(generation) changed = ( prev is None or prev.window_id != win.window_id @@ -265,6 +414,31 @@ def capture_frame(self) -> tuple["Image.Image", bool]: ) return image, changed + def _publish_locked(self, generation: int) -> None: + """Expose one queued frame geometry to action observers.""" + if generation != self._geometry_generation or self._window is None: + raise WindowCaptureError( + f"cannot publish geometry generation {generation}; " + f"the current generation is {self._geometry_generation}" + ) + self._published_generation = generation + self._published_window = self._window + self._published_scale_x = self._scale_x + self._published_scale_y = self._scale_y + self._published_content_rect = self._content_rect + + def publish_frame(self, generation: int) -> None: + """Publish geometry after its pixels and metadata enter the queue.""" + with self._lock: + self._publish_locked(generation) + + def current_generation(self) -> int: + """Return the most recently captured, not necessarily published, epoch.""" + with self._lock: + if self._geometry_generation < 1: + raise WindowCaptureError("no captured geometry generation is available") + return self._geometry_generation + def translate(self, x: float, y: float) -> tuple[float, float]: """Translate a global screen point into window-relative pixels. @@ -276,19 +450,52 @@ def translate(self, x: float, y: float) -> tuple[float, float]: :meth:`capture_frame` (no bounds are known yet, and guessing a coordinate space would be a silent wrong action). """ + px, py, _generation = self.translate_with_generation(x, y) + return px, py + + def _geometry_for_action( + self, + ) -> tuple[TargetWindow, float, float, tuple[int, int, int, int], int]: + """Return the published geometry after exact live revalidation.""" + self._assert_display_topology() with self._lock: - window = self._window - scale_x = self._scale_x - scale_y = self._scale_y - content_rect = self._content_rect + window = self._published_window + scale_x = self._published_scale_x + scale_y = self._published_scale_y + content_rect = self._published_content_rect + generation = self._published_generation if window is None or scale_x is None or scale_y is None or content_rect is None: raise WindowCaptureError( - "translate() called before the first captured frame; " + "an action arrived before the first published frame; " "capture_frame() must succeed before input can be scoped" ) + live = self.resolve() + self._assert_bound_identity(live) + if not live.on_screen: + raise WindowCaptureError("the target window is not on screen at action time") + if live.bounds != window.bounds: + raise WindowCaptureError( + "the target moved or resized after the last published frame; " + "wait for a matching frame before recording input" + ) + self._assert_display_topology() + return window, scale_x, scale_y, content_rect, generation + + def generation_for_action(self) -> int: + """Bind a non-pointer action to the exact published frame epoch.""" + return self._geometry_for_action()[4] + + def assert_current(self) -> None: + """Revalidate the bound process, bounds, and display topology.""" + self._geometry_for_action() + + def translate_with_generation(self, x: float, y: float) -> tuple[float, float, int]: + """Translate against the exact published frame after revalidation.""" + window, scale_x, scale_y, content_rect, generation = self._geometry_for_action() return ( (x - window.bounds[0]) * scale_x + content_rect[0], (y - window.bounds[1]) * scale_y + content_rect[1], + generation, ) def window_event_data(self) -> dict: @@ -308,9 +515,34 @@ def window_event_data(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + generation = self._geometry_generation + topology = self._display_topology if window is None: raise WindowCaptureError("no resolved window; call capture_frame() first") x, y, w, h = window.bounds + state = { + "schema_version": WINDOW_CAPTURE_SCHEMA_VERSION, + "window_capture": True, + "window_id": str(window.window_id), + "owner": window.owner, + "pid": window.pid, + "process_start_time": window.process_start_time, + "coordinate_source": window.coordinate_source, + "geometry_generation": generation, + "display_topology_sha256": ( + topology.get("topology_sha256") if topology else None + ), + "bounds": [x, y, w, h], + "scale": scale, + "scale_x": scale_x, + "scale_y": scale_y, + "viewport": list(viewport) if viewport else None, + "source_viewport": list(source_viewport) if source_viewport else None, + "content_rect": list(content_rect) if content_rect else None, + "fit_scale": fit_scale, + "on_screen": window.on_screen, + } + state["geometry_epoch_sha256"] = window_geometry_epoch_sha256(state) return { "title": window.title, "left": int(x), @@ -318,20 +550,7 @@ def window_event_data(self) -> dict: "width": int(w), "height": int(h), "window_id": str(window.window_id), - "state": { - "window_capture": True, - "owner": window.owner, - "pid": window.pid, - "bounds": [x, y, w, h], - "scale": scale, - "scale_x": scale_x, - "scale_y": scale_y, - "viewport": list(viewport) if viewport else None, - "source_viewport": (list(source_viewport) if source_viewport else None), - "content_rect": list(content_rect) if content_rect else None, - "fit_scale": fit_scale, - "on_screen": window.on_screen, - }, + "state": state, } def snapshot(self) -> dict: @@ -350,9 +569,13 @@ def snapshot(self) -> dict: source_viewport = self._source_viewport content_rect = self._content_rect fit_scale = self._fit_scale + generation = self._geometry_generation + topology = self._display_topology data: dict = { + "schema_version": WINDOW_CAPTURE_SCHEMA_VERSION, "target": {"owner": self.target.owner, "title": self.target.title}, "coordinate_space": "window_pixels", + "display_topology": topology, } if window is not None: data.update( @@ -361,6 +584,9 @@ def snapshot(self) -> dict: "owner": window.owner, "title": window.title, "pid": window.pid, + "process_start_time": window.process_start_time, + "coordinate_source": window.coordinate_source, + "geometry_generation": generation, "initial_bounds": list(window.bounds), "scale": scale, "scale_x": scale_x, @@ -401,6 +627,18 @@ def capture_window(window: TargetWindow) -> "Image.Image": ) +def _process_start_time(pid: int) -> float: + """Return a stable process creation identity or fail closed.""" + import psutil + + try: + return float(psutil.Process(pid).create_time()) + except psutil.Error as exc: + raise WindowCaptureError( + f"could not bind window owner PID {pid} to its process start time" + ) from exc + + def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: """macOS: CGWindowList by owner/title substring. @@ -435,13 +673,16 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: area = bounds[2] * bounds[3] if area > best_area: best_area = area + pid = int(w.get("kCGWindowOwnerPID", 0) or 0) best = TargetWindow( window_id=int(w.get("kCGWindowNumber", 0) or 0), owner=owner, title=name, - pid=int(w.get("kCGWindowOwnerPID", 0) or 0), + pid=pid, bounds=bounds, on_screen=bool(w.get("kCGWindowIsOnscreen", False)), + process_start_time=_process_start_time(pid), + coordinate_source="quartz-screen-points", ) return best @@ -519,9 +760,11 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: pid = wintypes.DWORD() user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) try: - proc_name = psutil.Process(pid.value).name() + process = psutil.Process(pid.value) + proc_name = process.name() + process_start_time = float(process.create_time()) except psutil.Error: - proc_name = "" + return True if owner_l is not None and owner_l not in proc_name.lower(): return True rect = _window_rect(hwnd) @@ -541,6 +784,8 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: float(bottom - top), ), on_screen=True, + process_start_time=process_start_time, + coordinate_source="dwm-physical-pixels", ) ) return True @@ -552,24 +797,23 @@ def _enum_cb(hwnd: int, _lparam: int) -> bool: def _window_rect(hwnd: int) -> tuple[int, int, int, int] | None: - """Window rectangle in screen coordinates (DWM extended frame preferred).""" + """Return DWM physical bounds; never mix DPI-virtualized coordinates.""" import ctypes import ctypes.wintypes as wintypes rect = wintypes.RECT() try: - DWMWA_EXTENDED_FRAME_BOUNDS = 9 - res = ctypes.windll.dwmapi.DwmGetWindowAttribute( - wintypes.HWND(hwnd), - ctypes.wintypes.DWORD(DWMWA_EXTENDED_FRAME_BOUNDS), - ctypes.byref(rect), - ctypes.sizeof(rect), - ) - if res == 0: - return (rect.left, rect.top, rect.right, rect.bottom) - except (AttributeError, OSError): # pragma: no cover - dwmapi always present - pass - if not ctypes.windll.user32.GetWindowRect(wintypes.HWND(hwnd), ctypes.byref(rect)): + dwmapi = ctypes.windll.dwmapi + except (AttributeError, OSError): # pragma: no cover - supported Windows has DWM + return None + DWMWA_EXTENDED_FRAME_BOUNDS = 9 + res = dwmapi.DwmGetWindowAttribute( + wintypes.HWND(hwnd), + ctypes.wintypes.DWORD(DWMWA_EXTENDED_FRAME_BOUNDS), + ctypes.byref(rect), + ctypes.sizeof(rect), + ) + if res != 0: return None return (rect.left, rect.top, rect.right, rect.bottom) diff --git a/tests/test_capture_terminal.py b/tests/test_capture_terminal.py new file mode 100644 index 0000000..d3549ba --- /dev/null +++ b/tests/test_capture_terminal.py @@ -0,0 +1,129 @@ +"""Immutable capture terminal and verified-consumer snapshot tests.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from openadapt_capture.capture import CaptureSession +from openadapt_capture.db import create_db, crud +from openadapt_capture.terminal import ( + ARTIFACT_MANIFEST_FILENAME, + CAPTURE_TERMINAL_FILENAME, + CaptureSealError, + seal_capture, + verify_capture_artifacts, +) + + +def _capture_directory(root: Path) -> Path: + capture_dir = root / "capture" + capture_dir.mkdir(parents=True) + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + crud.insert_recording( + session, + { + "timestamp": 10.0, + "monitor_width": 800, + "monitor_height": 600, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "sealed capture", + }, + ) + session.close() + engine.dispose() + (capture_dir / "artifact.bin").write_bytes(b"artifact") + (capture_dir / "capture-state.json").write_text('{"phase":"finalizing"}\n') + return capture_dir + + +def _seal(capture_dir: Path): + return seal_capture( + capture_dir, + session_id="session-1", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=12.0, + event_counts={ + "action": 0, + "screen": 0, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=None, + ) + + +def test_terminal_binds_canonical_manifest_bytes_including_newline(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + terminal = _seal(capture_dir) + manifest_raw = (capture_dir / ARTIFACT_MANIFEST_FILENAME).read_bytes() + + assert manifest_raw.endswith(b"\n") + assert terminal.artifact_manifest_size_bytes == len(manifest_raw) + assert terminal.artifact_manifest_sha256 == hashlib.sha256( + b"openadapt.capture-artifact-manifest.v1\0" + manifest_raw + ).hexdigest() + assert (capture_dir / CAPTURE_TERMINAL_FILENAME).read_bytes().endswith(b"\n") + verified_terminal, manifest = verify_capture_artifacts(capture_dir) + assert verified_terminal == terminal + assert [artifact.path for artifact in manifest.artifacts] == [ + "artifact.bin", + "recording.db", + ] + + +def test_mutable_control_state_is_not_part_of_the_immutable_inventory(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + _seal(capture_dir) + + (capture_dir / "capture-state.json").write_text('{"phase":"complete"}\n') + + verify_capture_artifacts(capture_dir) + + +def test_terminal_rejects_artifact_tamper_and_uninventoried_files(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + _seal(capture_dir) + (capture_dir / "artifact.bin").write_bytes(b"changed") + with pytest.raises(CaptureSealError, match="differs from its seal"): + verify_capture_artifacts(capture_dir) + + other = _capture_directory(tmp_path / "other") + _seal(other) + (other / "late.txt").write_text("late") + with pytest.raises(CaptureSealError, match="sealed inventory"): + verify_capture_artifacts(other) + + +def test_manifest_rejects_symbolic_links(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + (capture_dir / "linked.bin").symlink_to(capture_dir / "artifact.bin") + + with pytest.raises(CaptureSealError, match="not a regular file"): + _seal(capture_dir) + + +def test_verified_loader_uses_a_private_snapshot_without_migrating_source(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + terminal = _seal(capture_dir) + source_db = capture_dir / "recording.db" + before = (source_db.stat().st_mtime_ns, hashlib.sha256(source_db.read_bytes()).hexdigest()) + + with CaptureSession.load_verified(capture_dir) as capture: + assert capture.task_description == "sealed capture" + assert capture.capture_dir != capture_dir + assert capture.capture_dir.parent != capture_dir.parent + assert json.loads( + (capture.capture_dir / CAPTURE_TERMINAL_FILENAME).read_text() + )["terminal_sha256"] == terminal.terminal_sha256 + + after = (source_db.stat().st_mtime_ns, hashlib.sha256(source_db.read_bytes()).hexdigest()) + assert after == before diff --git a/tests/test_desktop_capture.py b/tests/test_desktop_capture.py index 0b455f8..2783c5c 100644 --- a/tests/test_desktop_capture.py +++ b/tests/test_desktop_capture.py @@ -58,7 +58,9 @@ def test_live_scope_rejects_same_size_origin_and_layout_change() -> None: def test_multiple_monitor_snapshot_is_privacy_safe_geometry() -> None: - assert _two_monitor_scope().snapshot() == { + snapshot = _two_monitor_scope().snapshot() + assert snapshot == { + "schema_version": "openadapt.capture.display-topology/v1", "coordinate_space": "virtual_desktop_pixels", "origin": [-1920, 0], "viewport": [4480, 1440], @@ -67,7 +69,9 @@ def test_multiple_monitor_snapshot_is_privacy_safe_geometry() -> None: [-1920, 0, 1920, 1080], [0, 0, 2560, 1440], ], + "topology_sha256": snapshot["topology_sha256"], } + assert len(snapshot["topology_sha256"]) == 64 def test_desktop_scope_rejects_missing_physical_monitor() -> None: diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 0783851..04bbff8 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -26,7 +26,13 @@ from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud -from openadapt_capture.recorder import Recorder, read_screen_events +from openadapt_capture.desktop_capture import DesktopCaptureScope +from openadapt_capture.recorder import ( + OrderedEventJournal, + Recorder, + WindowScopedFrame, + read_screen_events, +) from openadapt_capture.window_capture import ( TargetWindow, WindowCaptureError, @@ -138,6 +144,7 @@ def __init__(self, bounds=(300.0, 150.0, 800.0, 600.0), scale=2.0): self.window_id = 42 self.title = "Fake Window" self.missing = False + self.process_start_time = 123.5 def resolver(self, target: WindowTarget): if self.missing: @@ -148,6 +155,8 @@ def resolver(self, target: WindowTarget): title=self.title, pid=1234, bounds=self.bounds, + process_start_time=self.process_start_time, + coordinate_source="test-screen-points", ) def capturer(self, window: TargetWindow) -> Image.Image: @@ -163,11 +172,19 @@ def fake(): @pytest.fixture def scope(fake): - return WindowCaptureScope( + result = WindowCaptureScope( WindowTarget(owner="FakeApp"), resolver=fake.resolver, capturer=fake.capturer, ) + result.bind_display_topology( + { + "schema_version": "openadapt.capture.display-topology/v1", + "topology_sha256": "a" * 64, + }, + lambda **_kwargs: None, + ) + return result class TestWindowCaptureScope: @@ -227,10 +244,17 @@ def test_resize_uses_exact_axis_scales_after_integer_rounding(self, fake): ] ) scope = WindowCaptureScope( - WindowTarget(owner="Parallels"), + WindowTarget(owner="FakeApp"), resolver=fake.resolver, capturer=lambda _window: next(images), ) + scope.bind_display_topology( + { + "schema_version": "openadapt.capture.display-topology/v1", + "topology_sha256": "a" * 64, + }, + lambda **_kwargs: None, + ) scope.capture_frame() scope.capture_frame() @@ -260,9 +284,10 @@ def test_resolve_does_not_mix_new_bounds_with_previous_frame(self, scope, fake): fake.bounds = (100.0, 50.0, 800.0, 600.0) scope.resolve() - # A resolver poll alone cannot commit geometry. Translation changes - # only after the corresponding frame has been captured. - assert scope.translate(310.0, 170.0) == (20.0, 40.0) + # A resolver poll alone cannot commit geometry. Input stops until a + # frame with the new bounds is captured and published. + with pytest.raises(WindowCaptureError, match="moved or resized"): + scope.translate(310.0, 170.0) scope.capture_frame() assert scope.translate(310.0, 170.0) == (420.0, 240.0) @@ -290,6 +315,28 @@ def test_screen_reader_propagates_capture_failure_without_retry(self, scope, fak window_scope=scope, ) + def test_screen_reader_retains_terminal_frame_after_input_finishes(self, scope): + journal = OrderedEventJournal() + terminate = threading.Event() + terminate.set() + input_finished = threading.Event() + input_finished.set() + + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + input_finished=input_finished, + ) + + event = journal.get_nowait() + assert event.source_ordinal == 1 + assert isinstance(event.data, WindowScopedFrame) + assert event.data.geometry_generation == 1 + assert journal.empty() + def test_window_event_data_matches_window_event_columns(self, scope): scope.capture_frame() data = scope.window_event_data() @@ -429,6 +476,7 @@ def test_key_action_unaffected(self, scope): from openadapt_capture.recorder import trigger_action_event utils.set_start_time() + scope.capture_frame() q = queue.Queue() trigger_action_event(q, {"name": "press", "key_char": "a"}, scope) (event,) = self._drain(q) @@ -775,7 +823,7 @@ class TestWindowCaptureLive: def _scope(self) -> WindowCaptureScope: scope = WindowCaptureScope(WindowTarget(owner=_SMOKE_OWNER, title=_SMOKE_TITLE)) try: - scope.resolve() + resolved = scope.resolve() except WindowCaptureError as exc: if _PRODUCTION_QUALIFICATION: raise AssertionError( @@ -787,6 +835,12 @@ def _scope(self) -> WindowCaptureScope: f"title {_SMOKE_TITLE!r} on this desktop; open one (or set " "OPENADAPT_WINDOW_SMOKE_OWNER) to run the live smoke test" ) + if not resolved.on_screen: + if _PRODUCTION_QUALIFICATION: + raise AssertionError("the production qualification window is not on screen") + pytest.skip("the matching live smoke-test window is not on screen") + desktop = DesktopCaptureScope.current() + scope.bind_display_topology(desktop.snapshot(), desktop.assert_current) return scope def test_live_window_frame_and_translation(self): @@ -885,5 +939,12 @@ def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): def test_live_missing_window_fails_loud(self): scope = WindowCaptureScope(WindowTarget(owner="no-such-app-obviously-not-running-xyz")) + scope.bind_display_topology( + { + "schema_version": "openadapt.capture.display-topology/v1", + "topology_sha256": "a" * 64, + }, + lambda **_kwargs: None, + ) with pytest.raises(WindowCaptureError, match="no window matching"): scope.capture_frame() From 9b062adca7331c3843a01838c14dc47c505024c5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:33:10 -0400 Subject: [PATCH 2/6] fix: validate sealed geometry evidence --- openadapt_capture/capture.py | 235 ++++++++++++++++++++++++-- openadapt_capture/db/__init__.py | 4 +- openadapt_capture/events.py | 7 + openadapt_capture/recorder.py | 250 +++++++++++++--------------- openadapt_capture/terminal.py | 137 ++++++++++++++- openadapt_capture/video.py | 142 +++++++++++++--- openadapt_capture/window_capture.py | 10 +- tests/test_capture_terminal.py | 120 +++++++++++++ tests/test_frame_binding.py | 131 +++++++++++++-- tests/test_highlevel.py | 90 +++++++++- tests/test_video.py | 10 +- tests/test_window_capture.py | 159 ++++++++++++++++++ 12 files changed, 1099 insertions(+), 196 deletions(-) diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 6d600fe..52244fe 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -513,10 +513,191 @@ def screenshot(self) -> "Image" | None: """ bound = getattr(self.event, "screenshot_timestamp", None) if bound is not None: - return self._capture.get_exact_frame(bound) + return self._capture.get_exact_frame( + bound, + source_ordinal=getattr(self.event, "screenshot_source_ordinal", None), + ) return self._capture.get_frame_at(self.timestamp) +def _source_order(rows: list) -> list: + """Order current journal rows by ordinal and legacy rows by timestamp.""" + return sorted( + rows, + key=lambda row: ( + getattr(row, "source_ordinal", None) is None, + getattr(row, "source_ordinal", None) + if getattr(row, "source_ordinal", None) is not None + else row.timestamp, + ), + ) + + +def _validate_database_contract( + capture: "CaptureSession", + *, + event_counts: dict[str, int], + last_source_ordinal: int | None, +) -> None: + """Validate the sealed terminal claims against one immutable database.""" + recording = capture._recording + rows_by_kind = { + "action": list(recording.action_events), + "screen": list(recording.screenshots), + "window": list(recording.window_events), + "browser": list(recording.browser_events), + } + actual_counts = {kind: len(rows) for kind, rows in rows_by_kind.items()} + for kind, actual in actual_counts.items(): + if event_counts.get(kind) != actual: + raise InvalidCaptureEvent( + f"sealed {kind} count {event_counts.get(kind)!r} differs from database {actual}" + ) + for event in rows_by_kind["browser"]: + if _convert_browser_event(event) is None: + raise InvalidCaptureEvent("sealed capture has an invalid browser event") + + ordinals_by_kind: dict[str, set[int]] = {} + all_ordinals: list[int] = [] + for kind, rows in rows_by_kind.items(): + ordinals = [getattr(row, "source_ordinal", None) for row in rows] + if any(ordinal is None or ordinal <= 0 for ordinal in ordinals): + raise InvalidCaptureEvent(f"sealed {kind} rows have missing source ordinals") + typed_ordinals = [int(ordinal) for ordinal in ordinals] + if len(typed_ordinals) != len(set(typed_ordinals)): + raise InvalidCaptureEvent(f"sealed {kind} rows reuse a source ordinal") + ordinals_by_kind[kind] = set(typed_ordinals) + all_ordinals.extend(typed_ordinals) + + calculated_last = max(all_ordinals, default=None) + if calculated_last != last_source_ordinal: + raise InvalidCaptureEvent( + "sealed last source ordinal differs from the immutable database" + ) + + for event in rows_by_kind["action"]: + _convert_action_event(event) + if event.screenshot_id is None or event.screenshot is None: + raise InvalidCaptureEvent("sealed action has no retained screenshot relationship") + if ( + event.screenshot_source_ordinal != event.screenshot.source_ordinal + or event.screenshot_timestamp != event.screenshot.timestamp + or event.source_ordinal <= event.screenshot_source_ordinal + ): + raise InvalidCaptureEvent("sealed action screenshot relationship is inconsistent") + if event.window_event_source_ordinal is not None: + if event.window_event_id is None or event.window_event is None: + raise InvalidCaptureEvent("sealed action has no retained window relationship") + if ( + event.window_event_source_ordinal != event.window_event.source_ordinal + or event.window_event_timestamp != event.window_event.timestamp + ): + raise InvalidCaptureEvent("sealed action window relationship is inconsistent") + metadata = capture.window_capture + is_v2 = ( + isinstance(metadata, dict) + and metadata.get("schema_version") == "openadapt.capture.window-scoped/v2" + ) + allowed_pair_ordinals: set[int] = set() + if is_v2: + window_events = capture.window_capture_events_v2() + windows_by_ordinal = {event.source_ordinal: event for event in window_events} + screenshots_by_ordinal = { + row.source_ordinal: row for row in rows_by_kind["screen"] + } + if set(windows_by_ordinal) != set(screenshots_by_ordinal): + raise InvalidCaptureEvent("sealed v2 frame/window pairs are not a bijection") + allowed_pair_ordinals = set(windows_by_ordinal) + identity = None + topology_sha256 = None + generation_to_epoch: dict[int, str] = {} + previous_generation = 0 + for ordinal in sorted(allowed_pair_ordinals): + window_event = windows_by_ordinal[ordinal] + state = window_event.window_capture_v2 + assert state is not None + screenshot = screenshots_by_ordinal[ordinal] + if screenshot.timestamp != window_event.timestamp: + raise InvalidCaptureEvent("sealed v2 frame and geometry timestamps differ") + if screenshot.png_data: + from PIL import Image + + try: + with Image.open(io.BytesIO(screenshot.png_data)) as retained: + retained.load() + retained_size = retained.size + except Exception as exc: + raise InvalidCaptureEvent("sealed v2 frame PNG is invalid") from exc + if retained_size != state.viewport: + raise InvalidCaptureEvent( + "sealed v2 frame dimensions differ from its geometry viewport" + ) + current_identity = ( + state.window_id, + state.pid, + state.process_start_time, + state.owner.casefold(), + ) + if identity is None: + identity = current_identity + topology_sha256 = state.display_topology_sha256 + elif current_identity != identity: + raise InvalidCaptureEvent("sealed v2 capture changed window identity") + if state.display_topology_sha256 != topology_sha256: + raise InvalidCaptureEvent("sealed v2 capture changed display topology") + known_epoch = generation_to_epoch.setdefault( + state.geometry_generation, + state.geometry_epoch_sha256, + ) + if known_epoch != state.geometry_epoch_sha256: + raise InvalidCaptureEvent("one v2 generation names different geometry epochs") + if state.geometry_generation < previous_generation: + raise InvalidCaptureEvent("sealed v2 geometry generations move backwards") + previous_generation = state.geometry_generation + for action in rows_by_kind["action"]: + if ( + action.window_geometry_generation is None + or action.screenshot_source_ordinal + != action.window_event_source_ordinal + ): + raise InvalidCaptureEvent("sealed v2 action has an incomplete frame binding") + bound = windows_by_ordinal.get(action.window_event_source_ordinal) + if bound is None or ( + bound.window_capture_v2.geometry_generation + != action.window_geometry_generation + ): + raise InvalidCaptureEvent("sealed v2 action names the wrong geometry epoch") + + owners: dict[int, set[str]] = {} + for kind, ordinals in ordinals_by_kind.items(): + for ordinal in ordinals: + owners.setdefault(ordinal, set()).add(kind) + for ordinal, kinds in owners.items(): + if len(kinds) > 1 and not ( + ordinal in allowed_pair_ordinals and kinds == {"screen", "window"} + ): + raise InvalidCaptureEvent("sealed journal reuses a source ordinal across events") + + expected_video_count = event_counts.get("video") + if not isinstance(expected_video_count, int) or expected_video_count < 0: + raise InvalidCaptureEvent("sealed video count is invalid") + if expected_video_count: + from openadapt_capture.video import _read_timing_metadata + + video_path = capture.video_path + if video_path is None: + raise InvalidCaptureEvent("sealed capture claims video frames without an MP4") + timing = _read_timing_metadata(video_path) + if timing is None or timing[3] is None: + raise InvalidCaptureEvent("sealed MP4 has no source-ordinal frame bindings") + if len(timing[3]) != expected_video_count: + raise InvalidCaptureEvent("sealed video count differs from its MP4 bindings") + elif capture.video_path is not None: + raise InvalidCaptureEvent("sealed capture inventories an MP4 but claims no video frames") + + list(capture.actions(include_moves=True)) + + class CaptureSession: """A loaded capture session for analysis and replay. @@ -610,16 +791,28 @@ def _discard() -> None: temporary.cleanup() try: - recording = session.query(Recording).first() + recordings = session.query(Recording).all() except Exception: _discard() raise - if recording is None: + if len(recordings) != 1: _discard() - raise FileNotFoundError(f"Invalid capture (no recording found): {capture_dir}") + raise InvalidCaptureEvent( + f"verified capture must contain exactly one recording, found {len(recordings)}" + ) + recording = recordings[0] result = cls(snapshot_dir, session, recording) result._verified_tempdir = temporary result._verified_terminal = terminal + try: + _validate_database_contract( + result, + event_counts=terminal.event_counts.model_dump(mode="python"), + last_source_ordinal=terminal.last_source_ordinal, + ) + except BaseException: + result.close() + raise return result @property @@ -780,7 +973,7 @@ def raw_events(self) -> list[PydanticActionEvent]: List of raw mouse and keyboard events. """ events = [] - for db_event in self._recording.action_events: + for db_event in _source_order(list(self._recording.action_events)): if getattr(db_event, "disabled", False): continue events.append(_convert_action_event(db_event)) @@ -789,7 +982,7 @@ def raw_events(self) -> list[PydanticActionEvent]: def window_events(self) -> list[CapturedWindowEvent]: """Return stored window rows through the public validated event view.""" result: list[CapturedWindowEvent] = [] - for row in self._recording.window_events: + for row in _source_order(list(self._recording.window_events)): state = getattr(row, "state", None) if not isinstance(state, dict): raise InvalidCaptureEvent( @@ -818,7 +1011,7 @@ def frames(self) -> list[CapturedFrame]: source_ordinal=getattr(row, "source_ordinal", None), png_sha256=getattr(row, "png_sha256", None), ) - for row in self._recording.screenshots + for row in _source_order(list(self._recording.screenshots)) ] def window_capture_events_v2(self) -> list[CapturedWindowEvent]: @@ -930,7 +1123,12 @@ def get_frame_at(self, timestamp: float, tolerance: float = 0.5) -> "Image" | No except Exception: return None - def get_exact_frame(self, capture_timestamp: float) -> "Image": + def get_exact_frame( + self, + capture_timestamp: float, + *, + source_ordinal: int | None = None, + ) -> "Image": """Decode THE retained frame bound to this exact capture timestamp. Prefers the video's capture-timeline binding; for image-only captures @@ -947,17 +1145,30 @@ def get_exact_frame(self, capture_timestamp: float) -> "Image": if video_path is not None: from openadapt_capture.video import extract_exact_frame - return extract_exact_frame(video_path, capture_timestamp) + return extract_exact_frame( + video_path, + capture_timestamp, + source_ordinal=source_ordinal, + ) for screenshot in self._recording.screenshots: - if screenshot.timestamp == capture_timestamp: + ordinal_matches = ( + source_ordinal is None + or getattr(screenshot, "source_ordinal", None) == source_ordinal + ) + if screenshot.timestamp == capture_timestamp and ordinal_matches: if not screenshot.png_data: raise LookupError( f"the screenshot retained at {capture_timestamp!r} has no image data" ) return Image.open(io.BytesIO(screenshot.png_data)).convert("RGB") raise LookupError( - f"no frame was retained at exactly {capture_timestamp!r} " - "(fail-closed; refusing a nearest-frame substitute)" + f"no frame was retained at exactly {capture_timestamp!r}" + + ( + f" with source ordinal {source_ordinal} " + if source_ordinal is not None + else " " + ) + + "(fail-closed; refusing a nearest-frame substitute)" ) def close(self) -> None: diff --git a/openadapt_capture/db/__init__.py b/openadapt_capture/db/__init__.py index 5173008..f5b10bb 100644 --- a/openadapt_capture/db/__init__.py +++ b/openadapt_capture/db/__init__.py @@ -174,11 +174,11 @@ def get_session_for_path(db_path: str, echo: bool = False): def get_immutable_session_for_path(db_path: str, echo: bool = False): """Open an already-verified SQLite snapshot without schema migration.""" - resolved = str(Path(db_path).resolve()) + immutable_uri = f"{Path(db_path).resolve().as_uri()}?mode=ro&immutable=1" def _connect() -> sqlite3.Connection: return sqlite3.connect( - f"file:{resolved}?mode=ro&immutable=1", + immutable_uri, uri=True, check_same_thread=False, ) diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index b248198..907892d 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -147,6 +147,13 @@ def _closed_geometry(self) -> "WindowCaptureStateV2": ) if not math.isclose(self.fit_scale, expected_fit) or self.content_rect != expected_rect: raise ValueError("window capture normalization differs from its viewports") + expected_scale_x = width / self.bounds[2] + expected_scale_y = height / self.bounds[3] + if not math.isclose(self.scale_x, expected_scale_x) or not math.isclose( + self.scale_y, + expected_scale_y, + ): + raise ValueError("window capture axis scales differ from its content geometry") if not math.isclose(self.scale, self.scale_x): raise ValueError("legacy window scale differs from exact x scale") if self.geometry_epoch_sha256 != window_geometry_epoch_sha256( diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index e99a01f..b8459f9 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -188,7 +188,6 @@ class OrderedEventJournal: def __init__(self) -> None: self._condition = threading.Condition() self._entries: deque[_JournalEntry] = deque() - self._last_timestamp: float | None = None self._next_sequence = 1 def reserve(self, timestamp: float) -> EventReservation: @@ -196,17 +195,45 @@ def reserve(self, timestamp: float) -> EventReservation: if not math.isfinite(timestamp): raise EventJournalOrderingError("event timestamps must be finite") with self._condition: - if self._last_timestamp is not None and timestamp < self._last_timestamp: - raise EventJournalOrderingError( - "an event arrived behind a newer journal reservation" - ) - entry = _JournalEntry(timestamp, self._next_sequence) - self._next_sequence += 1 - self._last_timestamp = timestamp - self._entries.append(entry) - self._condition.notify_all() + entry = self._reserve_locked(timestamp) return EventReservation(self, entry) + def _reserve_locked(self, timestamp: float) -> _JournalEntry: + entry = _JournalEntry(timestamp, self._next_sequence) + self._next_sequence += 1 + self._entries.append(entry) + self._condition.notify_all() + return entry + + def reserve_window_action( + self, + timestamp: float, + window_scope: WindowCaptureScope, + x: float | None, + y: float | None, + ) -> tuple[EventReservation, tuple[float, float, int] | int]: + """Reserve an action and bind the last published frame atomically.""" + timestamp = float(timestamp) + if not math.isfinite(timestamp): + raise EventJournalOrderingError("event timestamps must be finite") + with self._condition: + entry = self._reserve_locked(timestamp) + reservation = EventReservation(self, entry) + try: + if x is not None and y is not None: + binding: tuple[float, float, int] | int = ( + window_scope.translate_with_generation(x, y) + ) + else: + binding = window_scope.generation_for_action() + except BaseException as exc: + entry.error = exc + entry.ready = True + reservation._finished = True + self._condition.notify_all() + raise + return reservation, binding + def put(self, event: Event, block: bool = True, timeout: float | None = None) -> None: del block, timeout reservation = self.reserve(event.timestamp) @@ -224,13 +251,8 @@ def commit_window_frame( raise EventJournalOrderingError("event timestamps must be finite") failure: BaseException | None = None with self._condition: - if self._last_timestamp is not None and timestamp < self._last_timestamp: - raise EventJournalOrderingError( - "a frame arrived behind a newer journal reservation" - ) entry = _JournalEntry(timestamp, self._next_sequence) self._next_sequence += 1 - self._last_timestamp = timestamp self._entries.append(entry) try: window_scope.publish_frame(generation) @@ -634,7 +656,7 @@ def processing_complete() -> bool: num_video_events.value += 1 if scoped_pair: process_event( - event, + event if config.RECORD_IMAGES else event._replace(data=None), screen_write_q, write_screen_event, recording, @@ -726,7 +748,11 @@ def processing_complete() -> bool: ) if screen_is_new: process_event( - prev_screen_event, + ( + prev_screen_event + if config.RECORD_IMAGES + else prev_screen_event._replace(data=None) + ), screen_write_q, write_screen_event, recording, @@ -798,6 +824,8 @@ def write_screen_event( recording: Recording, event: Event, perf_q: sq.SynchronizedQueue, + *, + record_images: bool | None = None, ) -> None: """Write a screen event to the database and update the performance queue. @@ -808,8 +836,11 @@ def write_screen_event( perf_q: A queue for collecting performance data. """ assert event.type == "screen", event + retain_image = config.RECORD_IMAGES if record_images is None else record_images image = event.data - if config.RECORD_IMAGES: + if retain_image: + if image is None: + raise ValueError("the screen writer received no image while PNG retention is enabled") with io.BytesIO() as output: image.save(output, format="PNG") png_data = output.getvalue() @@ -1075,7 +1106,8 @@ def write_video_event( # TODO: why isn't force_key_frame sufficient? if last_pts != 0: num_copies = 1 - for _ in range(num_copies): + for copy_index in range(num_copies): + bind_evidence = copy_index == 0 last_pts = video.write_video_frame( video_container, video_stream, @@ -1084,6 +1116,8 @@ def write_video_event( video_start_timestamp, last_pts, force_key_frame, + source_ordinal=event.source_ordinal if bind_evidence else None, + bind_capture=bind_evidence, ) perf_q.put((event.type, event.timestamp, utils.get_timestamp())) return { @@ -1124,20 +1158,43 @@ def trigger_action_event( None """ event_timestamp = utils.get_timestamp() if timestamp is None else timestamp - reservation = ( - event_q.reserve(event_timestamp) if isinstance(event_q, OrderedEventJournal) else None - ) event_data = dict(action_event_args) x = event_data.get("mouse_x") y = event_data.get("mouse_y") + window_binding: tuple[float, float, int] | int | None = None + if isinstance(event_q, OrderedEventJournal) and isinstance( + coordinate_scope, + WindowCaptureScope, + ): + reservation, window_binding = event_q.reserve_window_action( + event_timestamp, + coordinate_scope, + x, + y, + ) + else: + reservation = ( + event_q.reserve(event_timestamp) + if isinstance(event_q, OrderedEventJournal) + else None + ) try: if isinstance(coordinate_scope, WindowCaptureScope): if x is not None and y is not None: - wx, wy, generation = coordinate_scope.translate_with_generation(x, y) + if window_binding is None: + wx, wy, generation = coordinate_scope.translate_with_generation(x, y) + else: + assert isinstance(window_binding, tuple) + wx, wy, generation = window_binding event_data["mouse_x"] = wx event_data["mouse_y"] = wy else: - generation = coordinate_scope.generation_for_action() + generation = ( + coordinate_scope.generation_for_action() + if window_binding is None + else window_binding + ) + assert isinstance(generation, int) event_data["window_geometry_generation"] = generation elif coordinate_scope is not None and x is not None and y is not None: wx, wy = coordinate_scope.translate(x, y) @@ -2201,7 +2258,7 @@ def record( target=utils.WrapStdout(write_events), args=( "screen", - write_screen_event, + partial(write_screen_event, record_images=bool(config.RECORD_IMAGES)), screen_write_q, num_screen_events, perf_q, @@ -2734,8 +2791,8 @@ def _verify_completed_capture(self) -> None: from openadapt_capture.capture import ( CaptureSession, - _convert_action_event, - _convert_browser_event, + InvalidCaptureEvent, + _validate_database_contract, ) db_path = Path(self.capture_dir) / "recording.db" @@ -2792,109 +2849,34 @@ def _verify_completed_capture(self) -> None: database.close() capture = CaptureSession.load(self.capture_dir) try: - for event in capture._recording.action_events: - _convert_action_event(event) - for event in capture._recording.browser_events: - if _convert_browser_event(event) is None: - raise RuntimeError( - "The finalized Capture database has an invalid browser event." - ) - metadata = capture.window_capture - if ( - isinstance(metadata, dict) - and metadata.get("schema_version") == "openadapt.capture.window-scoped/v2" - ): - window_events = capture.window_capture_events_v2() - windows_by_ordinal = {event.source_ordinal: event for event in window_events} - screenshots_by_ordinal = { - row.source_ordinal: row for row in capture._recording.screenshots - } - if None in windows_by_ordinal or None in screenshots_by_ordinal: - raise RuntimeError("The finalized v2 capture has an unsequenced frame pair.") - if set(windows_by_ordinal) != set(screenshots_by_ordinal): - raise RuntimeError( - "The finalized v2 capture has an incomplete frame/window pair." - ) - if len(windows_by_ordinal) != len(window_events): - raise RuntimeError("The finalized v2 capture reuses a window source ordinal.") - identity = None - topology_sha256 = None - generation_to_epoch: dict[int, str] = {} - previous_generation = 0 - for ordinal in sorted(windows_by_ordinal): - window_event = windows_by_ordinal[ordinal] - window_state = window_event.window_capture_v2 - assert window_state is not None - screenshot = screenshots_by_ordinal[ordinal] - if screenshot.timestamp != window_event.timestamp: - raise RuntimeError("The finalized v2 frame and geometry timestamps differ.") - current_identity = ( - window_state.window_id, - window_state.pid, - window_state.process_start_time, - window_state.owner.casefold(), - ) - if identity is None: - identity = current_identity - topology_sha256 = window_state.display_topology_sha256 - elif current_identity != identity: - raise RuntimeError( - "The finalized v2 capture changed process-bound window identity." - ) - if window_state.display_topology_sha256 != topology_sha256: - raise RuntimeError("The finalized v2 capture changed display topology.") - generation = window_state.geometry_generation - known_epoch = generation_to_epoch.setdefault( - generation, window_state.geometry_epoch_sha256 - ) - if known_epoch != window_state.geometry_epoch_sha256: - raise RuntimeError("A v2 geometry generation names two different epochs.") - if generation < previous_generation: - raise RuntimeError("The finalized v2 geometry generations move backwards.") - previous_generation = generation - action_ordinals: set[int] = set() - for action in capture._recording.action_events: - if ( - action.source_ordinal is None - or action.screenshot_source_ordinal is None - or action.window_event_source_ordinal is None - or action.window_geometry_generation is None - ): - raise RuntimeError( - "The finalized v2 capture has an incomplete action binding." - ) - if action.source_ordinal in action_ordinals: - raise RuntimeError( - "The finalized v2 capture reuses an action source ordinal." - ) - action_ordinals.add(action.source_ordinal) - if ( - action.screenshot_source_ordinal != action.window_event_source_ordinal - or action.source_ordinal <= action.screenshot_source_ordinal - ): - raise RuntimeError( - "The finalized v2 action does not bind one earlier frame pair." - ) - bound_window = windows_by_ordinal.get(action.window_event_source_ordinal) - bound_screen = screenshots_by_ordinal.get(action.screenshot_source_ordinal) - if bound_window is None or bound_screen is None: - raise RuntimeError("The finalized v2 action names a missing frame pair.") - bound_state = bound_window.window_capture_v2 - assert bound_state is not None - if ( - bound_state.geometry_generation != action.window_geometry_generation - or action.screenshot_id != bound_screen.id - or action.window_event_id - != next( - row.id - for row in capture._recording.window_events - if row.source_ordinal == action.window_event_source_ordinal - ) - ): - raise RuntimeError( - "The finalized v2 action relationship differs from its epoch." - ) - list(capture.actions(include_moves=True)) + database_rows = ( + list(capture._recording.action_events) + + list(capture._recording.screenshots) + + list(capture._recording.window_events) + + list(capture._recording.browser_events) + ) + last_source_ordinal = max( + ( + row.source_ordinal + for row in database_rows + if row.source_ordinal is not None + ), + default=None, + ) + try: + _validate_database_contract( + capture, + event_counts={ + "action": self._num_action_events.value, + "screen": self._num_screen_events.value, + "window": self._num_window_events.value, + "browser": self._num_browser_events.value, + "video": self._num_video_events.value, + }, + last_source_ordinal=last_source_ordinal, + ) + except InvalidCaptureEvent as exc: + raise RuntimeError(str(exc)) from exc finally: capture.close() @@ -3172,7 +3154,7 @@ def is_recording(self) -> bool: return ( self._record_thread is not None and self._record_thread.is_alive() - and not self._terminate_processing.is_set() + and not self._finalized_event.is_set() ) @property @@ -3213,12 +3195,14 @@ def capture(self): Returns None if recording has not finished yet. """ + if not self._finalized_event.is_set(): + return None self.check_health() - if self._capture is None and not self.is_recording: + if self._capture is None: try: from openadapt_capture.capture import CaptureSession - self._capture = CaptureSession.load(self.capture_dir) + self._capture = CaptureSession.load_verified(self.capture_dir) except FileNotFoundError: return None return self._capture diff --git a/openadapt_capture/terminal.py b/openadapt_capture/terminal.py index d655523..14da0ee 100644 --- a/openadapt_capture/terminal.py +++ b/openadapt_capture/terminal.py @@ -181,6 +181,82 @@ def _open_stable_regular_file(path: Path) -> tuple[int, os.stat_result]: return fd, before +def _open_relative_regular_file( + root: Path, + relative_path: str, +) -> tuple[int, os.stat_result]: + """Open one artifact without following any intermediate path component.""" + record = ArtifactRecord(path=relative_path, size_bytes=0, sha256="0" * 64) + parts = PurePosixPath(record.path).parts + supports_descriptor_walk = ( + os.open in os.supports_dir_fd + and os.stat in os.supports_dir_fd + and hasattr(os, "O_DIRECTORY") + ) + if not supports_descriptor_walk: + return _open_stable_regular_file(_safe_artifact_path(root, record.path)) + + directory_flags = os.O_RDONLY | os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + directory_flags |= os.O_NOFOLLOW + directory_fd = os.open(root, directory_flags) + opened_directories = [directory_fd] + try: + try: + for part in parts[:-1]: + directory_fd = os.open(part, directory_flags, dir_fd=directory_fd) + opened_directories.append(directory_fd) + before = os.stat(parts[-1], dir_fd=directory_fd, follow_symlinks=False) + if not stat.S_ISREG(before.st_mode): + raise CaptureSealError( + f"capture artifact is not a regular file: {record.path}" + ) + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(parts[-1], flags, dir_fd=directory_fd) + opened = os.fstat(fd) + if not stat.S_ISREG(opened.st_mode) or ( + opened.st_dev, + opened.st_ino, + ) != (before.st_dev, before.st_ino): + os.close(fd) + raise CaptureSealError( + f"capture artifact changed before reading: {record.path}" + ) + return fd, before + except OSError as exc: + raise CaptureSealError( + f"capture artifact path changed before reading: {record.path}" + ) from exc + finally: + for opened_directory in reversed(opened_directories): + os.close(opened_directory) + + +def _assert_relative_identity( + root: Path, + relative_path: str, + expected: os.stat_result, +) -> None: + fd, current = _open_relative_regular_file(root, relative_path) + os.close(fd) + if ( + expected.st_dev, + expected.st_ino, + expected.st_size, + expected.st_mtime_ns, + ) != ( + current.st_dev, + current.st_ino, + current.st_size, + current.st_mtime_ns, + ): + raise CaptureSealError( + f"capture artifact changed while reading: {relative_path}" + ) + + def _assert_stable_file(path: Path, before: os.stat_result, after: os.stat_result) -> None: current = path.lstat() identity = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) @@ -222,6 +298,38 @@ def _hash_regular_file(path: Path) -> tuple[int, str]: return size, digest.hexdigest() +def _hash_relative_regular_file(root: Path, relative_path: str) -> tuple[int, str]: + fd, before = _open_relative_regular_file(root, relative_path) + try: + digest = hashlib.sha256() + size = 0 + while True: + chunk = os.read(fd, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + after = os.fstat(fd) + finally: + os.close(fd) + if ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): + raise CaptureSealError(f"capture artifact changed while reading: {relative_path}") + _assert_relative_identity(root, relative_path, before) + if size != before.st_size: + raise CaptureSealError(f"capture artifact size changed while hashing: {relative_path}") + return size, digest.hexdigest() + + def _read_regular_file(path: Path) -> bytes: """Read one stable regular file through the descriptor that was verified.""" fd, before = _open_stable_regular_file(path) @@ -244,12 +352,13 @@ def _read_regular_file(path: Path) -> bytes: def _copy_verified_regular_file( - source: Path, + source_root: Path, + relative_path: str, destination: Path, expected: ArtifactRecord, ) -> None: """Copy one exact source file without following a replacement symlink.""" - source_fd, before = _open_stable_regular_file(source) + source_fd, before = _open_relative_regular_file(source_root, relative_path) flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) try: destination_fd = os.open(destination, flags, 0o600) @@ -278,7 +387,22 @@ def _copy_verified_regular_file( except BaseException: destination.unlink(missing_ok=True) raise - _assert_stable_file(source, before, after) + if ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): + destination.unlink(missing_ok=True) + raise CaptureSealError( + f"capture artifact changed during snapshot: {relative_path}" + ) + _assert_relative_identity(source_root, relative_path, before) if (size, digest.hexdigest()) != (expected.size_bytes, expected.sha256): destination.unlink(missing_ok=True) raise CaptureSealError(f"capture artifact changed during snapshot: {expected.path}") @@ -297,7 +421,7 @@ def build_artifact_manifest(capture_dir: str | os.PathLike[str]) -> CaptureArtif continue if not stat.S_ISREG(details.st_mode): raise CaptureSealError(f"capture artifact is not a regular file: {relative}") - size, digest = _hash_regular_file(path) + size, digest = _hash_relative_regular_file(root, relative) artifacts.append(ArtifactRecord(path=relative, size_bytes=size, sha256=digest)) return CaptureArtifactManifest( schema_version=ARTIFACT_MANIFEST_SCHEMA_VERSION, @@ -401,7 +525,7 @@ def verify_capture_artifacts( if actual_paths != expected_paths: raise CaptureSealError("capture artifacts differ from the sealed inventory") for artifact in manifest.artifacts: - size, digest = _hash_regular_file(_safe_artifact_path(root, artifact.path)) + size, digest = _hash_relative_regular_file(root, artifact.path) if (size, digest) != (artifact.size_bytes, artifact.sha256): raise CaptureSealError(f"capture artifact differs from its seal: {artifact.path}") return terminal, manifest @@ -417,10 +541,9 @@ def copy_verified_capture( destination = Path(temporary.name) try: for artifact in manifest.artifacts: - source_path = _safe_artifact_path(source, artifact.path) target_path = destination.joinpath(*PurePosixPath(artifact.path).parts) target_path.parent.mkdir(parents=True, exist_ok=True) - _copy_verified_regular_file(source_path, target_path, artifact) + _copy_verified_regular_file(source, artifact.path, target_path, artifact) size, digest = _hash_regular_file(target_path) if (size, digest) != (artifact.size_bytes, artifact.sha256): raise CaptureSealError(f"capture artifact changed during snapshot: {artifact.path}") diff --git a/openadapt_capture/video.py b/openadapt_capture/video.py index e0418aa..51ab5bd 100644 --- a/openadapt_capture/video.py +++ b/openadapt_capture/video.py @@ -89,6 +89,7 @@ def _append_timing_box( fps: Fraction, frames: list[tuple[int, float]], captures: list[tuple[int, float]] | None = None, + sources: list[tuple[int, int]] | None = None, ) -> None: """Append logical capture-frame timestamps in an ignored MP4 UUID box. @@ -105,6 +106,8 @@ def _append_timing_box( } if captures: payload["captures"] = [[index, timestamp] for index, timestamp in captures] + if sources: + payload["sources"] = [[index, ordinal] for index, ordinal in sources] serialized = json.dumps(payload, separators=(",", ":")).encode("utf-8") box_size = 24 + len(serialized) if len(serialized) > _MAX_TIMING_PAYLOAD_BYTES or box_size >= 2**32: @@ -116,12 +119,17 @@ def _append_timing_box( os.fsync(output.fileno()) -def _read_timing_box( +def _read_timing_metadata( path: Path, -) -> tuple[Fraction, list[tuple[int, float]], list[tuple[int, float]] | None]: +) -> tuple[ + Fraction, + list[tuple[int, float]], + list[tuple[int, float]] | None, + list[tuple[int, int]] | None, +]: """Read OpenAdapt logical timestamps from top-level MP4 boxes, if present. - Returns ``(fps, frames, captures)``. ``captures`` is ``None`` for media + Returns ``(fps, frames, captures, sources)``. ``captures`` is ``None`` for media recorded before exact frame binding existed; consumers that require the exact retained frame must treat that as unavailable (fail closed). """ @@ -229,11 +237,50 @@ def _read_timing_box( captures.append((entry[0], bound_timestamp)) capture_index = entry[0] capture_timestamp = bound_timestamp - return fps, result, captures + sources: list[tuple[int, int]] | None = None + raw_sources = payload.get("sources") + if raw_sources is not None: + sources = [] + retained_indexes = {index for index, _ in result} + source_index = -1 + source_ordinal = 0 + for entry in raw_sources: + if ( + not isinstance(entry, list) + or len(entry) != 2 + or not isinstance(entry[0], int) + or not isinstance(entry[1], int) + ): + raise FFmpegEncodingError( + "Video timing metadata has an invalid source binding" + ) + if entry[0] <= source_index or entry[1] <= source_ordinal: + raise FFmpegEncodingError( + "Video timing metadata source bindings are not ordered" + ) + if entry[0] not in retained_indexes: + raise FFmpegEncodingError( + "Video timing metadata source binding names no retained frame" + ) + sources.append((entry[0], entry[1])) + source_index = entry[0] + source_ordinal = entry[1] + return fps, result, captures, sources offset += box_size return None +def _read_timing_box( + path: Path, +) -> tuple[Fraction, list[tuple[int, float]], list[tuple[int, float]] | None] | None: + """Read the legacy three-part timing view used by existing callers.""" + timing = _read_timing_metadata(path) + if timing is None: + return None + fps, frames, captures, _ = timing + return fps, frames, captures + + def _validate_option_token(label: str, value: str) -> str: if not _OPTION_TOKEN.fullmatch(value): raise FFmpegUnavailableError( @@ -730,6 +777,7 @@ def __init__( self._emitted_frames = 0 self._logical_frames: list[tuple[int, float]] = [] self._capture_frames: list[tuple[int, float]] = [] + self._source_frames: list[tuple[int, int]] = [] self._closed = False self._lock = threading.Lock() @@ -891,6 +939,7 @@ def stage_frame( image: "PILImage", pts: int, capture_timestamp: float | None = None, + source_ordinal: int | None = None, ) -> None: """Stream one frame, filling PTS gaps deterministically without disk. @@ -933,6 +982,19 @@ def stage_frame( "Capture timestamp must be a finite wall-clock value" ) self._capture_frames.append((encoded_index, bound_timestamp)) + if source_ordinal is not None: + if ( + not isinstance(source_ordinal, int) + or source_ordinal <= 0 + or ( + self._source_frames + and source_ordinal <= self._source_frames[-1][1] + ) + ): + raise FFmpegEncodingError( + "Source ordinals must be positive and strictly ordered" + ) + self._source_frames.append((encoded_index, source_ordinal)) self._emitted_frames += emitted self._last_frame = frame self._last_pts = pts @@ -1000,12 +1062,18 @@ def close(self) -> None: fps=self.stream.average_rate, frames=self._logical_frames, captures=self._capture_frames, + sources=self._source_frames, ) - _decode_first_frame_png( + verification_png = _decode_first_frame_png( self.provision, self.partial_path, timeout=min(self.timeout_seconds, EXTRACT_TIMEOUT_SECONDS), ) + with Image.open(io.BytesIO(verification_png)) as decoded: + if decoded.size != (self.stream.width, self.stream.height): + raise FFmpegEncodingError( + "Decoded video dimensions differ from the fixed capture viewport" + ) os.replace(self.partial_path, self.output_path) except BaseException: self._abort_and_reap() @@ -1087,6 +1155,7 @@ def write_frame( image: "PILImage", timestamp: float, force_key_frame: bool = False, + source_ordinal: int | None = None, ) -> None: del force_key_frame # FFmpeg makes the first encoded frame a key frame. with self._lock: @@ -1099,6 +1168,7 @@ def write_frame( timestamp, self._start_time, self._last_pts, + source_ordinal=source_ordinal, ) def close(self) -> None: @@ -1182,13 +1252,21 @@ def write_video_frame( video_start_timestamp: float, last_pts: int, force_key_frame: bool = False, + *, + source_ordinal: int | None = None, + bind_capture: bool = True, ) -> int: del force_key_frame time_diff = max(timestamp - video_start_timestamp, 0.0) pts = int(time_diff * float(video_stream.average_rate)) if pts <= last_pts: pts = last_pts + 1 - video_container.stage_frame(screenshot, pts, capture_timestamp=timestamp) + video_container.stage_frame( + screenshot, + pts, + capture_timestamp=timestamp if bind_capture else None, + source_ordinal=source_ordinal, + ) return pts @@ -1210,6 +1288,7 @@ def finalize_video_writer( video_start_timestamp, last_pts, force_key_frame=True, + bind_capture=False, ) video_container.close() if fix_moov: @@ -1224,7 +1303,7 @@ def move_moov_atom( ) -> None: provision = resolve_ffmpeg(ffmpeg_path or config.VIDEO_FFMPEG_PATH) input_path = Path(input_file) - timing = _read_timing_box(input_path) + timing = _read_timing_metadata(input_path) temp_file: Path | None = None if output_file is None: temp_file = input_path.with_name(f".{input_path.name}.{uuid.uuid4().hex}.mp4") @@ -1250,8 +1329,14 @@ def move_moov_atom( timeout=DEFAULT_PROCESS_TIMEOUT_SECONDS, ) if timing is not None: - fps, logical_frames, captures = timing - _append_timing_box(output_path, fps=fps, frames=logical_frames, captures=captures) + fps, logical_frames, captures, sources = timing + _append_timing_box( + output_path, + fps=fps, + frames=logical_frames, + captures=captures, + sources=sources, + ) if temp_file is not None: os.replace(temp_file, input_path) @@ -1414,6 +1499,7 @@ def extract_exact_frame( video_path: str | Path, capture_timestamp: float, *, + source_ordinal: int | None = None, ffmpeg_path: str | os.PathLike[str] | None = None, ffprobe_path: str | os.PathLike[str] | None = None, ) -> "PILImage": @@ -1427,25 +1513,43 @@ def extract_exact_frame( at. """ path = Path(video_path) - timing = _read_timing_box(path) + timing = _read_timing_metadata(path) if timing is None: raise LookupError( f"{path}: no OpenAdapt timing metadata; " "the exact retained frame cannot be resolved (fail-closed)" ) - _, _, captures = timing - if not captures: + _, _, captures, sources = timing + if source_ordinal is not None: + if not sources: + raise LookupError( + f"{path}: timing metadata has no source-ordinal bindings; " + "the exact retained frame cannot be resolved (fail-closed)" + ) + matches = [index for index, bound in sources if bound == source_ordinal] + if len(matches) != 1: + raise LookupError( + f"{path}: expected one retained frame bound to source ordinal " + f"{source_ordinal}, found {len(matches)} (fail-closed)" + ) + elif not captures: raise LookupError( f"{path}: timing metadata predates exact frame binding; " "the exact retained frame cannot be resolved (fail-closed)" ) - matches = [index for index, bound in captures if bound == capture_timestamp] - if not matches: - nearest = min(captures, key=lambda item: abs(item[1] - capture_timestamp)) - raise LookupError( - f"{path}: no retained frame bound to {capture_timestamp!r} " - f"(nearest binding: index {nearest[0]} at {nearest[1]!r})" - ) + else: + matches = [index for index, bound in captures if bound == capture_timestamp] + if not matches: + nearest = min(captures, key=lambda item: abs(item[1] - capture_timestamp)) + raise LookupError( + f"{path}: no retained frame bound to {capture_timestamp!r} " + f"(nearest binding: index {nearest[0]} at {nearest[1]!r})" + ) + if len(matches) != 1: + raise LookupError( + f"{path}: capture timestamp {capture_timestamp!r} has " + f"{len(matches)} frame bindings (fail-closed)" + ) provision = resolve_ffmpeg( ffmpeg_path or config.VIDEO_FFMPEG_PATH, ffprobe_path or config.VIDEO_FFPROBE_PATH, diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 5760a96..22e030d 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -280,6 +280,8 @@ def resolve(self) -> TargetWindow: raise WindowCaptureError( "the window resolver returned a title outside the configured selector" ) + if not win.on_screen: + raise WindowCaptureError("the resolved target window is not on screen") if win.pid <= 0: raise WindowCaptureError("the resolved target has no owning process identity") if ( @@ -642,10 +644,8 @@ def _process_start_time(pid: int) -> float: def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: """macOS: CGWindowList by owner/title substring. - Same selection semantics as flow's ``MacWindowClient.find_window``: - ``kCGWindowListOptionAll`` (a momentarily hidden client is still - resolvable/capturable), layer 0 only, case-insensitive substring match, - largest window wins. + Selects the largest visible layer-0 window that matches the configured + owner/title substrings. """ import Quartz @@ -663,6 +663,8 @@ def _resolve_window_macos(target: WindowTarget) -> TargetWindow | None: continue if int(w.get("kCGWindowLayer", 0) or 0) != 0: continue # skip menubar/overlay layers; the app window is layer 0 + if not bool(w.get("kCGWindowIsOnscreen", False)): + continue b = w.get("kCGWindowBounds", {}) or {} bounds = ( float(b.get("X", 0.0)), diff --git a/tests/test_capture_terminal.py b/tests/test_capture_terminal.py index d3549ba..66b4000 100644 --- a/tests/test_capture_terminal.py +++ b/tests/test_capture_terminal.py @@ -8,6 +8,7 @@ import pytest +import openadapt_capture.terminal as terminal_module from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud from openadapt_capture.terminal import ( @@ -111,6 +112,37 @@ def test_manifest_rejects_symbolic_links(tmp_path) -> None: _seal(capture_dir) +def test_verifier_rejects_an_intermediate_directory_replaced_during_read( + tmp_path, + monkeypatch, +) -> None: + capture_dir = _capture_directory(tmp_path) + nested = capture_dir / "nested" + nested.mkdir() + (nested / "evidence.bin").write_bytes(b"evidence") + _seal(capture_dir) + moved = tmp_path / "moved-nested" + original_hash = terminal_module._hash_relative_regular_file + replaced = False + + def replace_then_hash(root, relative_path): + nonlocal replaced + if relative_path == "nested/evidence.bin" and not replaced: + replaced = True + nested.rename(moved) + nested.symlink_to(moved, target_is_directory=True) + return original_hash(root, relative_path) + + monkeypatch.setattr( + terminal_module, + "_hash_relative_regular_file", + replace_then_hash, + ) + + with pytest.raises(CaptureSealError, match="path changed"): + verify_capture_artifacts(capture_dir) + + def test_verified_loader_uses_a_private_snapshot_without_migrating_source(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) terminal = _seal(capture_dir) @@ -127,3 +159,91 @@ def test_verified_loader_uses_a_private_snapshot_without_migrating_source(tmp_pa after = (source_db.stat().st_mtime_ns, hashlib.sha256(source_db.read_bytes()).hexdigest()) assert after == before + + +def test_verified_loader_rejects_terminal_counts_that_differ_from_database(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + seal_capture( + capture_dir, + session_id="session-1", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=12.0, + event_counts={ + "action": 1, + "screen": 0, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=None, + ) + + with pytest.raises(ValueError, match="action count"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_multiple_recordings(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + crud.insert_recording( + session, + { + "timestamp": 20.0, + "monitor_width": 800, + "monitor_height": 600, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "second recording", + }, + ) + session.close() + engine.dispose() + _seal(capture_dir) + + with pytest.raises(ValueError, match="exactly one recording"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_duplicate_source_ordinals(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path) + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + recording = session.query(crud.Recording).one() + for timestamp in (11.0, 12.0): + crud.insert_screenshot( + session, + recording, + timestamp, + {"source_ordinal": 1, "png_sha256": None}, + ) + session.close() + engine.dispose() + seal_capture( + capture_dir, + session_id="session-1", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=12.0, + event_counts={ + "action": 0, + "screen": 2, + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=1, + ) + + with pytest.raises(ValueError, match="reuse a source ordinal"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_opens_an_encoded_immutable_database_uri(tmp_path) -> None: + capture_dir = _capture_directory(tmp_path / "capture#fragment") + _seal(capture_dir) + + with CaptureSession.load_verified(capture_dir) as capture: + assert capture.task_description == "sealed capture" diff --git a/tests/test_frame_binding.py b/tests/test_frame_binding.py index 5e30948..0c814a3 100644 --- a/tests/test_frame_binding.py +++ b/tests/test_frame_binding.py @@ -20,7 +20,8 @@ from PIL import Image from openadapt_capture import video -from openadapt_capture.capture import Action +from openadapt_capture.capture import Action, CaptureSession +from openadapt_capture.db import create_db, crud from openadapt_capture.events import ( KeyDownEvent, KeyTypeEvent, @@ -49,11 +50,13 @@ def test_timing_box_round_trips_exact_capture_bindings(tmp_path): path.write_bytes(b"\x00\x00\x00\x08ftyp") frames = [(0, 0.0), (3, 3 / 24), (4, 4 / 24)] captures = [(0, 1000.5), (3, 1001.25)] + sources = [(0, 1), (3, 4)] video._append_timing_box( path, fps=Fraction(24), frames=frames, captures=captures, + sources=sources, ) fps, read_frames, read_captures = video._read_timing_box(path) @@ -62,6 +65,7 @@ def test_timing_box_round_trips_exact_capture_bindings(tmp_path): # JSON float round-trip preserves the double exactly. assert read_captures == captures assert read_captures[1][1] == 1001.25 + assert video._read_timing_metadata(path)[3] == sources def test_timing_box_without_bindings_reads_as_none(tmp_path): @@ -178,10 +182,9 @@ def fake_extract(video_path, frame_index, provision): assert frame.size == (2, 1) -def test_extract_exact_frame_prefers_the_first_duplicate_binding( - tmp_path, monkeypatch +def test_extract_exact_frame_refuses_an_ambiguous_timestamp_binding( + tmp_path, ): - """The duplicated first frame binds twice; decode its first index.""" executable = tmp_path / "ffmpeg" executable.write_bytes(b"fake") path = tmp_path / "capture.mp4" @@ -192,14 +195,37 @@ def test_extract_exact_frame_prefers_the_first_duplicate_binding( frames=[(0, 0.0), (1, 1 / 24)], captures=[(0, 500.0), (1, 500.0)], ) + with pytest.raises(LookupError, match="2 frame bindings"): + video.extract_exact_frame(path, 500.0, ffmpeg_path=executable) + + +def test_extract_exact_frame_decodes_one_source_ordinal(tmp_path, monkeypatch): + executable = tmp_path / "ffmpeg" + executable.write_bytes(b"fake") + path = tmp_path / "capture.mp4" + path.write_bytes(b"\x00\x00\x00\x08ftyp") + video._append_timing_box( + path, + fps=Fraction(24), + frames=[(0, 0.0), (1, 1 / 24)], + captures=[(0, 500.0)], + sources=[(0, 7), (1, 9)], + ) decoded = {} monkeypatch.setattr( video, "_extract_frame_index_png", lambda _path, index, _provision: decoded.setdefault("index", index), ) - video.extract_exact_frame(path, 500.0, ffmpeg_path=executable) - assert decoded["index"] == 0 + + video.extract_exact_frame( + path, + 500.0, + source_ordinal=9, + ffmpeg_path=executable, + ) + + assert decoded["index"] == 1 # --------------------------------------------------------------------------- @@ -270,7 +296,7 @@ def _png_bytes(color: str = "black") -> bytes: import io output = io.BytesIO() - Image.new("RGB", (2, 2), color).save(output, format="PNG") + Image.new("RGB", (2, 1), color).save(output, format="PNG") return output.getvalue() @@ -297,13 +323,14 @@ def test_stage_frame_binds_only_newly_captured_frames(tmp_path, monkeypatch): blue = Image.new("RGB", (2, 1), "blue") # Frame at pts 27 fills three PTS slots (two gap fillers + itself). - stage.stage_frame(red, 24, capture_timestamp=111.0) - stage.stage_frame(blue, 27, capture_timestamp=113.5) + stage.stage_frame(red, 24, capture_timestamp=111.0, source_ordinal=1) + stage.stage_frame(blue, 27, capture_timestamp=113.5, source_ordinal=4) stage.close() _, logical, captures = video._read_timing_box(output) assert [index for index, _ in logical] == [0, 3] assert captures == [(0, 111.0), (3, 113.5)] + assert video._read_timing_metadata(output)[3] == [(0, 1), (3, 4)] def test_stage_frame_refuses_a_non_finite_capture_timestamp(tmp_path, monkeypatch): @@ -367,11 +394,16 @@ def test_merge_leaves_legacy_children_unbound(): class _StubCapture: def __init__(self): - self.exact_calls: list[float] = [] + self.exact_calls: list[tuple[float, int | None]] = [] self.lenient_calls: list[float] = [] - def get_exact_frame(self, capture_timestamp: float) -> Image.Image: - self.exact_calls.append(capture_timestamp) + def get_exact_frame( + self, + capture_timestamp: float, + *, + source_ordinal: int | None = None, + ) -> Image.Image: + self.exact_calls.append((capture_timestamp, source_ordinal)) return Image.new("RGB", (1, 1), "red") def get_frame_at(self, timestamp: float) -> Image.Image: @@ -384,11 +416,84 @@ def test_action_screenshot_uses_exact_binding(): _, up = _click_pair() action = Action(event=up, _capture=stub) image = action.screenshot - assert stub.exact_calls == [10.0] + assert stub.exact_calls == [(10.0, None)] assert stub.lenient_calls == [] assert image.getpixel((0, 0)) == (255, 0, 0) +def test_capture_session_joins_database_action_to_exact_mp4_source_frame( + tmp_path, + monkeypatch, +): + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + recording = crud.insert_recording( + session, + { + "timestamp": 1.0, + "monitor_width": 2, + "monitor_height": 1, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "exact media binding", + }, + ) + crud.insert_screenshot( + session, + recording, + 5.0, + {"source_ordinal": 7, "png_data": _png_bytes("red")}, + ) + crud.insert_action_event( + session, + recording, + 6.0, + { + "source_ordinal": 8, + "name": "click", + "mouse_x": 1, + "mouse_y": 1, + "mouse_button_name": "left", + "mouse_pressed": True, + "screenshot_timestamp": 5.0, + "screenshot_source_ordinal": 7, + }, + ) + session.close() + engine.dispose() + + video_path = capture_dir / "video.mp4" + video_path.write_bytes(b"\x00\x00\x00\x08ftyp") + video._append_timing_box( + video_path, + fps=Fraction(24), + frames=[(0, 0.0), (3, 3 / 24)], + captures=[(0, 4.0), (3, 5.0)], + sources=[(0, 3), (3, 7)], + ) + decoded = {} + monkeypatch.setattr(video, "resolve_ffmpeg", lambda *_args, **_kwargs: object()) + + def decode(_path, index, _provision): + decoded["index"] = index + return Image.new("RGB", (2, 1), "blue") + + monkeypatch.setattr( + video, + "_extract_frame_index_png", + decode, + ) + + with CaptureSession.load(capture_dir) as capture: + image = next(capture.actions(include_moves=True)).screenshot + + assert decoded["index"] == 3 + assert image.getpixel((0, 0)) == (0, 0, 255) + + def test_action_screenshot_falls_back_for_legacy_events(): stub = _StubCapture() legacy = MouseUpEvent(timestamp=42.0, x=0.0, y=0.0, button="left") diff --git a/tests/test_highlevel.py b/tests/test_highlevel.py index 20f7ee5..b884fb6 100644 --- a/tests/test_highlevel.py +++ b/tests/test_highlevel.py @@ -8,12 +8,14 @@ import threading import time from pathlib import Path +from types import SimpleNamespace import pytest +from PIL import Image from openadapt_capture import recorder as recorder_module from openadapt_capture import video -from openadapt_capture.capture import Capture, InvalidCaptureEvent +from openadapt_capture.capture import Capture, CaptureSession, InvalidCaptureEvent from openadapt_capture.db import create_db, crud from openadapt_capture.platform import DisplayMetricsUnavailable from openadapt_capture.recorder import Recorder @@ -28,6 +30,32 @@ _HELPER_ENGINES = [] +def _spawn_write_screen_with_explicit_retention( + db_path: str, + recording_id: int, + perf_queue, +) -> None: + """Spawn target whose imported config keeps the default image posture.""" + from openadapt_capture import utils + from openadapt_capture.db import get_session_for_path + from openadapt_capture.db.models import Recording + from openadapt_capture.recorder import Event, write_screen_event + + utils.set_start_time() + session = get_session_for_path(db_path) + try: + recording = session.get(Recording, recording_id) + write_screen_event( + session, + recording, + Event(2.0, "screen", Image.new("RGB", (2, 1), "blue"), 1), + perf_queue, + record_images=True, + ) + finally: + session.close() + + @pytest.fixture def temp_capture_dir(): """Create a temporary directory for captures.""" @@ -93,6 +121,46 @@ def test_recorder_accepts_capture_params(self): ) assert rec.task_description == "test" + def test_explicit_image_retention_crosses_the_spawn_boundary(self, tmp_path): + db_path = str(tmp_path / "recording.db") + engine, session_factory = create_db(db_path) + session = session_factory() + recording = crud.insert_recording( + session, + { + "timestamp": 1.0, + "monitor_width": 2, + "monitor_height": 1, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "test", + "task_description": "spawn override", + }, + ) + recording_id = recording.id + session.close() + engine.dispose() + context = multiprocessing.get_context("spawn") + perf_queue = context.Queue() + process = context.Process( + target=_spawn_write_screen_with_explicit_retention, + args=(db_path, recording_id, perf_queue), + ) + + process.start() + process.join(timeout=30) + + assert process.exitcode == 0 + engine, session_factory = create_db(db_path) + session = session_factory() + try: + screenshot = session.query(crud.Screenshot).one() + assert screenshot.source_ordinal == 1 + assert screenshot.png_data + finally: + session.close() + engine.dispose() + def test_recorder_event_count_property(self): """Test Recorder has event_count property starting at 0.""" rec = Recorder("/tmp/test_never_created") @@ -124,6 +192,26 @@ def test_recorder_capture_property_before_recording(self): rec = Recorder("/tmp/test_never_created") assert rec.capture is None + def test_recorder_capture_waits_for_seal_and_uses_verified_loader( + self, + monkeypatch, + tmp_path, + ): + rec = Recorder(str(tmp_path / "capture")) + rec._record_thread = SimpleNamespace(is_alive=lambda: True) + rec._terminate_processing.set() + assert rec.capture is None + + loaded = object() + monkeypatch.setattr( + CaptureSession, + "load_verified", + classmethod(lambda _cls, _path: loaded), + ) + rec._finalized_event.set() + + assert rec.capture is loaded + def test_recorder_screen_count_property(self): """Test Recorder has screen_count property.""" rec = Recorder("/tmp/test_never_created") diff --git a/tests/test_video.py b/tests/test_video.py index 91e7b0c..f509a4b 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -135,7 +135,7 @@ def popen(command, **kwargs): def _png_bytes(color: str = "black") -> bytes: output = io.BytesIO() - Image.new("RGB", (2, 2), color).save(output, format="PNG") + Image.new("RGB", (2, 1), color).save(output, format="PNG") return output.getvalue() @@ -962,9 +962,9 @@ def test_real_external_mpeg4_preserves_metadata_and_nearest_frame(tmp_path): Fraction(24), [(0, 0.0), (24, 1.0), (25, 25 / 24)], ) - # The duplicated first frame and the finalized last frame each bind to - # their exact capture wall-clock timestamps. - assert captures == [(0, start), (24, start + 1), (25, start + 1)] + # Finalization can extend playback, but retained evidence has one binding + # per source frame. + assert captures == [(0, start), (24, start + 1)] bound_frame = video.extract_exact_frame(output, start + 1, ffmpeg_path=executable) assert bound_frame.getpixel((10, 10))[2] > bound_frame.getpixel((10, 10))[0] frame = video.extract_frame( @@ -977,4 +977,4 @@ def test_real_external_mpeg4_preserves_metadata_and_nearest_frame(tmp_path): video.move_moov_atom(output, ffmpeg_path=executable) _, _, moved_captures = video._read_timing_box(output) - assert moved_captures == [(0, start), (24, start + 1), (25, start + 1)] + assert moved_captures == [(0, start), (24, start + 1)] diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 04bbff8..41d3b1d 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -24,10 +24,13 @@ import pytest from PIL import Image +import openadapt_capture.window_capture as window_capture_module from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud from openadapt_capture.desktop_capture import DesktopCaptureScope +from openadapt_capture.events import WindowCaptureStateV2, window_geometry_epoch_sha256 from openadapt_capture.recorder import ( + Event, OrderedEventJournal, Recorder, WindowScopedFrame, @@ -47,6 +50,160 @@ # --------------------------------------------------------------------------- +def test_journal_orders_concurrent_observations_by_reservation_not_timestamp(): + journal = OrderedEventJournal() + later_clock = journal.reserve(20.0) + earlier_clock = journal.reserve(10.0) + + earlier_clock.complete(Event(10.0, "action", {})) + later_clock.complete(Event(20.0, "action", {})) + + first = journal.get_nowait() + second = journal.get_nowait() + assert (first.timestamp, first.source_ordinal) == (20.0, 1) + assert (second.timestamp, second.source_ordinal) == (10.0, 2) + + +def test_journal_accepts_a_frame_after_a_process_clock_reset(scope): + journal = OrderedEventJournal() + first_image, _ = scope.capture_frame(publish=False) + first_generation = scope.current_generation() + journal.commit_window_frame( + Event( + 20.0, + "screen", + WindowScopedFrame( + image=first_image, + window_event_data=scope.window_event_data(), + geometry_generation=first_generation, + ), + ), + scope, + first_generation, + ) + + second_image, _ = scope.capture_frame(publish=False) + second_generation = scope.current_generation() + journal.commit_window_frame( + Event( + 10.0, + "screen", + WindowScopedFrame( + image=second_image, + window_event_data=scope.window_event_data(), + geometry_generation=second_generation, + ), + ), + scope, + second_generation, + ) + + assert [journal.get_nowait().source_ordinal for _ in range(2)] == [1, 2] + + +def test_action_reservation_cannot_bind_a_later_frame_generation(scope, fake, monkeypatch): + scope.capture_frame() + journal = OrderedEventJournal() + translation_entered = threading.Event() + allow_translation = threading.Event() + original_translate = scope.translate_with_generation + + def blocked_translate(x, y): + binding = original_translate(x, y) + translation_entered.set() + assert allow_translation.wait(timeout=5) + return binding + + monkeypatch.setattr(scope, "translate_with_generation", blocked_translate) + action_result = {} + + def reserve_action(): + reservation, binding = journal.reserve_window_action(1.0, scope, 310.0, 170.0) + action_result["binding"] = binding + reservation.complete(Event(1.0, "action", {"window_geometry_generation": binding[2]})) + + action_thread = threading.Thread(target=reserve_action) + action_thread.start() + assert translation_entered.wait(timeout=5) + + fake.bounds = (500.0, 250.0, 800.0, 600.0) + image, _ = scope.capture_frame(publish=False) + generation = scope.current_generation() + frame_thread = threading.Thread( + target=lambda: journal.commit_window_frame( + Event( + 2.0, + "screen", + WindowScopedFrame( + image=image, + window_event_data=scope.window_event_data(), + geometry_generation=generation, + ), + ), + scope, + generation, + ) + ) + frame_thread.start() + allow_translation.set() + action_thread.join(timeout=5) + frame_thread.join(timeout=5) + + assert action_result["binding"][2] == 1 + action = journal.get_nowait() + frame = journal.get_nowait() + assert (action.source_ordinal, frame.source_ordinal) == (1, 2) + assert frame.data.geometry_generation == 2 + + +def test_window_capture_state_rejects_scales_not_derived_from_content(scope): + scope.capture_frame() + state = scope.window_event_data()["state"] + state["scale"] = 999.0 + state["scale_x"] = 999.0 + state["scale_y"] = 999.0 + state["geometry_epoch_sha256"] = window_geometry_epoch_sha256( + {key: value for key, value in state.items() if key != "geometry_epoch_sha256"} + ) + + with pytest.raises(ValueError, match="axis scales"): + WindowCaptureStateV2.model_validate(state) + + +def test_macos_resolver_ignores_a_larger_hidden_matching_window(monkeypatch): + hidden = { + "kCGWindowOwnerName": "FakeApp", + "kCGWindowName": "Document", + "kCGWindowLayer": 0, + "kCGWindowIsOnscreen": False, + "kCGWindowBounds": {"X": 0, "Y": 0, "Width": 2000, "Height": 1200}, + "kCGWindowOwnerPID": 100, + "kCGWindowNumber": 1, + } + visible = { + "kCGWindowOwnerName": "FakeApp", + "kCGWindowName": "Document", + "kCGWindowLayer": 0, + "kCGWindowIsOnscreen": True, + "kCGWindowBounds": {"X": 10, "Y": 10, "Width": 800, "Height": 600}, + "kCGWindowOwnerPID": 100, + "kCGWindowNumber": 2, + } + quartz = SimpleNamespace( + kCGWindowListOptionAll=1, + kCGNullWindowID=0, + CGWindowListCopyWindowInfo=lambda *_args: [hidden, visible], + ) + monkeypatch.setitem(sys.modules, "Quartz", quartz) + monkeypatch.setattr(window_capture_module, "_process_start_time", lambda _pid: 123.0) + + resolved = window_capture_module._resolve_window_macos(WindowTarget(owner="FakeApp")) + + assert resolved is not None + assert resolved.window_id == 2 + assert resolved.on_screen is True + + class TestTranslatePoint: """Coordinate translation: global screen points -> window pixels.""" @@ -884,6 +1041,8 @@ def test_live_move_resize_preserves_fixed_viewport_and_restores_window(self): # an owner-only selector from switching to another large window after # the target changes size. scope = WindowCaptureScope(WindowTarget(owner=target.owner, title=target.title)) + desktop = DesktopCaptureScope.current() + scope.bind_display_topology(desktop.snapshot(), desktop.assert_current) initial_image, initial_changed = scope.capture_frame() assert initial_changed is True initial_data = scope.window_event_data() From 999e1412d8f828b635e064b97104941bf71ed52c Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:46:04 -0400 Subject: [PATCH 3/6] fix: serialize native frame observation --- openadapt_capture/processing.py | 15 ---- openadapt_capture/recorder.py | 111 +++++++++++++++------------- openadapt_capture/window_capture.py | 10 ++- tests/test_window_capture.py | 92 +++++++++++++++++++++++ 4 files changed, 162 insertions(+), 66 deletions(-) diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index b302e4f..1ae7034 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -90,21 +90,6 @@ def _first_structural_observation( return None -def _bound_screenshot_timestamp(events: list[ActionEvent]) -> float | None: - """Keep the frame bound where the merged action completed. - - The recorder binds an action to the screen frame retained when the action - was emitted (a click at button-up, a typed run at its last key), so the - merged event carries the LAST child's binding, not the first's. - """ - - bound: float | None = None - for event in events: - if event.screenshot_timestamp is not None: - bound = event.screenshot_timestamp - return bound - - def _merged_frame_binding(events: list[ActionEvent]) -> dict[str, float | int | None]: """Return the terminal child binding and reject a mixed native epoch.""" if not events: diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index b8459f9..67d022f 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -216,22 +216,23 @@ def reserve_window_action( timestamp = float(timestamp) if not math.isfinite(timestamp): raise EventJournalOrderingError("event timestamps must be finite") - with self._condition: - entry = self._reserve_locked(timestamp) - reservation = EventReservation(self, entry) - try: - if x is not None and y is not None: - binding: tuple[float, float, int] | int = ( - window_scope.translate_with_generation(x, y) - ) - else: - binding = window_scope.generation_for_action() - except BaseException as exc: - entry.error = exc - entry.ready = True - reservation._finished = True - self._condition.notify_all() - raise + with window_scope.observation_boundary(): + with self._condition: + entry = self._reserve_locked(timestamp) + reservation = EventReservation(self, entry) + try: + if x is not None and y is not None: + binding: tuple[float, float, int] | int = ( + window_scope.translate_with_generation(x, y) + ) + else: + binding = window_scope.generation_for_action() + except BaseException as exc: + entry.error = exc + entry.ready = True + reservation._finished = True + self._condition.notify_all() + raise return reservation, binding def put(self, event: Event, block: bool = True, timeout: float | None = None) -> None: @@ -250,19 +251,20 @@ def commit_window_frame( if not math.isfinite(timestamp): raise EventJournalOrderingError("event timestamps must be finite") failure: BaseException | None = None - with self._condition: - entry = _JournalEntry(timestamp, self._next_sequence) - self._next_sequence += 1 - self._entries.append(entry) - try: - window_scope.publish_frame(generation) - except BaseException as exc: - entry.error = exc - failure = exc - else: - entry.event = event._replace(source_ordinal=entry.sequence) - entry.ready = True - self._condition.notify_all() + with window_scope.observation_boundary(): + with self._condition: + entry = _JournalEntry(timestamp, self._next_sequence) + self._next_sequence += 1 + self._entries.append(entry) + try: + window_scope.publish_frame(generation) + except BaseException as exc: + entry.error = exc + failure = exc + else: + entry.event = event._replace(source_ordinal=entry.sequence) + entry.ready = True + self._condition.notify_all() if failure is not None: raise failure @@ -1427,11 +1429,35 @@ def capture_one() -> tuple[float, float]: nonlocal started t_start = time.perf_counter() if window_scope is not None: - # Any failed capture terminates the session. Retrying would omit a - # frame while input continues and could produce complete-looking - # evidence with a missing interval. - screenshot, _window_changed = window_scope.capture_frame(publish=False) - elif desktop_scope is not None: + with window_scope.observation_boundary(): + # Any failed capture terminates the session. Retrying would omit a + # frame while input continues and could produce complete-looking + # evidence with a missing interval. + screenshot, _window_changed = window_scope.capture_frame(publish=False) + t_screenshot = time.perf_counter() + if screenshot is None: + raise WindowCaptureError("the captured screenshot was empty") + if not started: + started_event.set() + started = True + frame_timestamp = utils.get_timestamp() + if not isinstance(event_q, OrderedEventJournal): + raise WindowCaptureError( + "window-scoped capture requires the ordered event journal" + ) + generation = window_scope.current_generation() + scoped_frame = WindowScopedFrame( + image=screenshot, + window_event_data=window_scope.window_event_data(), + geometry_generation=generation, + ) + event_q.commit_window_frame( + Event(frame_timestamp, "screen", scoped_frame), + window_scope, + generation, + ) + return t_start, t_screenshot + if desktop_scope is not None: # A monitor can move or change scale while the combined frame keeps # the same dimensions. Check both sides of the grab so neither the # frame nor later input uses stale origin or monitor geometry. @@ -1447,22 +1473,7 @@ def capture_one() -> tuple[float, float]: started_event.set() started = True frame_timestamp = utils.get_timestamp() - if window_scope is not None: - if not isinstance(event_q, OrderedEventJournal): - raise WindowCaptureError("window-scoped capture requires the ordered event journal") - generation = window_scope.current_generation() - scoped_frame = WindowScopedFrame( - image=screenshot, - window_event_data=window_scope.window_event_data(), - geometry_generation=generation, - ) - event_q.commit_window_frame( - Event(frame_timestamp, "screen", scoped_frame), - window_scope, - generation, - ) - else: - event_q.put(Event(frame_timestamp, "screen", screenshot)) + event_q.put(Event(frame_timestamp, "screen", screenshot)) return t_start, t_screenshot while not terminate_processing.is_set(): diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 22e030d..9eeca9e 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -38,8 +38,9 @@ import math import sys import threading +from contextlib import contextmanager from dataclasses import dataclass -from typing import TYPE_CHECKING, Callable, Optional +from typing import TYPE_CHECKING, Callable, Iterator, Optional from loguru import logger @@ -212,6 +213,7 @@ def __init__( self._resolver = resolver or resolve_window self._capturer = capturer or capture_window self._lock = threading.Lock() + self._observation_lock = threading.RLock() self._window: TargetWindow | None = None self._scale: float | None = None self._scale_x: float | None = None @@ -236,6 +238,12 @@ def __init__( # first frame's timeline entry. self._frame_window: TargetWindow | None = None + @contextmanager + def observation_boundary(self) -> Iterator[None]: + """Serialize a frame acquisition with native input observation.""" + with self._observation_lock: + yield + def bind_display_topology( self, snapshot: dict, diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 41d3b1d..3a43bb8 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -156,6 +156,98 @@ def reserve_action(): assert frame.data.geometry_generation == 2 +def test_input_observation_waits_for_an_in_flight_window_frame(fake): + capture_entered = threading.Event() + release_capture = threading.Event() + terminate = threading.Event() + block_capture = threading.Event() + + def capturer(window): + if block_capture.is_set(): + capture_entered.set() + assert release_capture.wait(timeout=5) + terminate.set() + return fake.capturer(window) + + scope = WindowCaptureScope( + WindowTarget(owner="FakeApp"), + resolver=fake.resolver, + capturer=capturer, + ) + scope.bind_display_topology( + { + "schema_version": "openadapt.capture.display-topology/v1", + "topology_sha256": "a" * 64, + }, + lambda **_kwargs: None, + ) + journal = OrderedEventJournal() + image, _ = scope.capture_frame(publish=False) + generation = scope.current_generation() + journal.commit_window_frame( + Event( + time.time(), + "screen", + WindowScopedFrame( + image=image, + window_event_data=scope.window_event_data(), + geometry_generation=generation, + ), + ), + scope, + generation, + ) + + block_capture.set() + screen_reader = threading.Thread( + target=read_screen_events, + args=( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + ), + kwargs={"window_scope": scope}, + ) + screen_reader.start() + assert capture_entered.wait(timeout=5) + + action_finished = threading.Event() + + def reserve_action(): + action_timestamp = time.time() + reservation, binding = journal.reserve_window_action( + action_timestamp, + scope, + 310.0, + 170.0, + ) + reservation.complete( + Event( + action_timestamp, + "action", + {"window_geometry_generation": binding[2]}, + ) + ) + action_finished.set() + + action_reader = threading.Thread(target=reserve_action) + action_reader.start() + assert not action_finished.wait(timeout=0.1) + + release_capture.set() + screen_reader.join(timeout=5) + action_reader.join(timeout=5) + + assert not screen_reader.is_alive() + assert not action_reader.is_alive() + assert [journal.get_nowait().type for _ in range(3)] == [ + "screen", + "screen", + "action", + ] + + def test_window_capture_state_rejects_scales_not_derived_from_content(scope): scope.capture_frame() state = scope.window_event_data()["state"] From 5c92e5666b387ea4913991abb8325f38ed0eb289 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 16:43:25 -0400 Subject: [PATCH 4/6] feat: capture exact Linux X11 window pixels --- README.md | 32 +- openadapt_capture/window_capture.py | 101 +++- openadapt_capture/window_capture_linux.py | 608 ++++++++++++++++++++++ pyproject.toml | 2 + tests/test_window_capture_linux.py | 446 ++++++++++++++++ 5 files changed, 1179 insertions(+), 10 deletions(-) create mode 100644 openadapt_capture/window_capture_linux.py create mode 100644 tests/test_window_capture_linux.py diff --git a/README.md b/README.md index 296d988..e4d2c26 100644 --- a/README.md +++ b/README.md @@ -219,8 +219,9 @@ retain window-scoped pixels and coordinates for Flow's remote visual compiler. **Status: implemented, with display-free unit coverage on every supported operating system.** The production release gate also requires live window capture, input injection, movement, resize, video verification, and no -skipped tests on interactive macOS and Windows runners. A customer RDP or -Citrix deployment still requires task- and environment-specific qualification. +skipped tests on interactive macOS and Windows runners. Linux X11 has a separate +opt-in live window check. A customer RDP or Citrix deployment still requires +task- and environment-specific qualification. By default the recorder captures the full screen. Window-scoped mode records ONE window in that window's own pixel space. This is the mode built for @@ -240,18 +241,20 @@ with Recorder( input("Perform the task, then press Enter...") ``` -`owner` matches the application (macOS: window owner name; Windows: process -executable name) and `title` optionally disambiguates among its windows; both -are case-insensitive substrings, mirroring how `openadapt-flow`'s -remote-display backend identifies the same window at replay time. The -selectors can also be set via config/environment +`owner` matches the application. macOS uses the window owner name. Windows and +Linux use the process executable name. `title` optionally disambiguates among +the application's windows. Both selectors are case-insensitive substrings, +matching how `openadapt-flow` identifies the same window at replay time. The +selectors can also be set through config or environment (`RECORD_WINDOW_OWNER` / `RECORD_WINDOW_TITLE`). In this mode: - **Frames are the target window's pixels.** macOS captures the window's own buffer (`CGWindowListCreateImage`, the identical call flow's replay uses); - Windows grabs the window's screen region, so keep the window unoccluded. + Linux X11 reads an XComposite named-window pixmap. It doesn't use a root + screenshot, so another window cannot replace the target pixels. Windows + grabs the window's screen region, so keep the window unoccluded. - **Input coordinates are translated at capture time** into the captured frame's pixel space (`pixel = (global_point - window_origin) * scale`, the exact inverse of the replay mapping). Input outside the window records @@ -280,6 +283,11 @@ In this mode: lost window, capture failure, or unexpected output-frame size fails the session instead of producing complete-looking media with an evidence gap. +Linux window mode requires an X11 session with EWMH and XComposite. Capture +won't start window mode in a native Wayland or XWayland-only session. A future +Wayland producer must bind the portal-selected window, its pixel stream, and +event-time coordinates before it can replace this refusal. + Note for converters: window-mode coordinates are already in captured-frame pixels (`coordinate_space == "window_pixels"`); do not rescale them by `pixel_ratio`. @@ -431,6 +439,14 @@ permissions: uv run pytest -m slow ``` +The Linux X11 window check also requires an explicit target: + +```bash +OPENADAPT_CAPTURE_LINUX_WINDOW_QUALIFICATION=1 \ +OPENADAPT_WINDOW_SMOKE_OWNER=citrix \ +uv run pytest tests/test_window_capture_linux.py -m slow +``` + ## License [MIT](LICENSE) diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 9eeca9e..f1d8e6b 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -302,8 +302,37 @@ def resolve(self) -> TargetWindow: ) if not win.coordinate_source.strip(): raise WindowCaptureError("the resolved target has no coordinate source") + self._assert_window_topology_compatibility(win) return win + def _assert_window_topology_compatibility(self, win: TargetWindow) -> None: + """Require an X11 root-pixel window to fit the retained monitor union.""" + if win.coordinate_source != "x11-root-physical-pixels": + return + with self._lock: + topology = self._display_topology + # A preliminary resolve can run before the recorder binds topology. + # Every frame and action resolve runs after that binding. + if topology is None: + return + if topology.get("coordinate_space") != "virtual_desktop_pixels": + raise WindowCaptureError("X11 window capture requires virtual_desktop_pixels topology") + monitors = topology.get("monitors") + if not isinstance(monitors, list) or not monitors: + raise WindowCaptureError("X11 window capture requires physical monitor bounds") + try: + monitor_rects = [tuple(float(value) for value in monitor) for monitor in monitors] + except (TypeError, ValueError) as exc: + raise WindowCaptureError( + "X11 display topology contains invalid monitor bounds" + ) from exc + if any(len(monitor) != 4 for monitor in monitor_rects): + raise WindowCaptureError("X11 display topology contains invalid monitor bounds") + if not _rectangle_covered_by_monitors(win.bounds, monitor_rects): + raise WindowCaptureError( + "the X11 target window is not fully covered by the bound display topology" + ) + def _assert_bound_identity(self, win: TargetWindow) -> None: """Reject a recycled window handle or a different owning process.""" with self._lock: @@ -615,14 +644,71 @@ def snapshot(self) -> dict: # --------------------------------------------------------------------------- +def _rectangle_covered_by_monitors( + rectangle: tuple[float, float, float, float], + monitors: list[tuple[float, float, float, float]], +) -> bool: + """Return whether the union of monitor rectangles fully covers a window.""" + x, y, width, height = rectangle + if width <= 0 or height <= 0: + return False + right = x + width + bottom = y + height + relevant = [ + monitor + for monitor in monitors + if monitor[2] > 0 + and monitor[3] > 0 + and monitor[0] < right + and monitor[1] < bottom + and monitor[0] + monitor[2] > x + and monitor[1] + monitor[3] > y + ] + if not relevant: + return False + x_edges = {x, right} + y_edges = {y, bottom} + for left, top, monitor_width, monitor_height in relevant: + x_edges.update({max(x, left), min(right, left + monitor_width)}) + y_edges.update({max(y, top), min(bottom, top + monitor_height)}) + sorted_x = sorted(x_edges) + sorted_y = sorted(y_edges) + for left, cell_right in zip(sorted_x, sorted_x[1:]): + if cell_right <= left: + continue + midpoint_x = (left + cell_right) / 2 + for top, cell_bottom in zip(sorted_y, sorted_y[1:]): + if cell_bottom <= top: + continue + midpoint_y = (top + cell_bottom) / 2 + if not any( + monitor_left <= midpoint_x < monitor_left + monitor_width + and monitor_top <= midpoint_y < monitor_top + monitor_height + for monitor_left, monitor_top, monitor_width, monitor_height in relevant + ): + return False + return True + + def resolve_window(target: WindowTarget) -> TargetWindow | None: """Find the front-most/largest window matching ``target`` on this platform.""" if sys.platform == "darwin": return _resolve_window_macos(target) if sys.platform == "win32": return _resolve_window_windows(target) + if sys.platform.startswith("linux"): + from openadapt_capture.window_capture_linux import ( + LinuxWindowCaptureError, + resolve_window_linux, + ) + + try: + return resolve_window_linux(target) + except LinuxWindowCaptureError as exc: + raise WindowCaptureError(str(exc)) from exc raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" + f"window-scoped capture is not supported on {sys.platform} " + "(supported: darwin, win32, linux-x11)" ) @@ -632,8 +718,19 @@ def capture_window(window: TargetWindow) -> "Image.Image": return _capture_window_macos(window) if sys.platform == "win32": return _capture_window_windows(window) + if sys.platform.startswith("linux"): + from openadapt_capture.window_capture_linux import ( + LinuxWindowCaptureError, + capture_window_linux, + ) + + try: + return capture_window_linux(window) + except LinuxWindowCaptureError as exc: + raise WindowCaptureError(str(exc)) from exc raise WindowCaptureError( - f"window-scoped capture is not supported on {sys.platform} (supported: darwin, win32)" + f"window-scoped capture is not supported on {sys.platform} " + "(supported: darwin, win32, linux-x11)" ) diff --git a/openadapt_capture/window_capture_linux.py b/openadapt_capture/window_capture_linux.py new file mode 100644 index 0000000..dde9b44 --- /dev/null +++ b/openadapt_capture/window_capture_linux.py @@ -0,0 +1,608 @@ +"""Fail-closed Linux X11 window resolution and XComposite pixel capture. + +The accessibility tree does not provide pixels. This producer uses EWMH/X11 +metadata to resolve one local top-level window and XComposite to name that +window's backing pixmap. It refuses native Wayland sessions because Capture +does not yet own a portal session that binds a selected window, its pixels, +and event-time coordinates. + +All XCB imports and display access stay inside call paths. Importing this +module is safe on a headless host and on non-Linux platforms. +""" + +from __future__ import annotations + +import os +from contextlib import AbstractContextManager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Protocol + +if TYPE_CHECKING: + from PIL import Image + + from openadapt_capture.window_capture import TargetWindow, WindowTarget + + +class LinuxWindowCaptureError(RuntimeError): + """The Linux session cannot produce an exact window-scoped frame.""" + + +@dataclass(frozen=True) +class X11WindowRecord: + """Process-neutral metadata read from one EWMH client window.""" + + window_id: int + title: str + pid: int + bounds: tuple[int, int, int, int] + viewable: bool + + +@dataclass(frozen=True) +class X11PixelFormat: + """The server format needed to decode one X11 ZPixmap reply.""" + + bits_per_pixel: int + scanline_pad: int + image_byte_order: int + red_mask: int + green_mask: int + blue_mask: int + alpha_mask: int = 0 + visual_class: int = 4 # X11 TrueColor + + +class _X11ClientProtocol(Protocol): + """Small injectable surface used by headless tests.""" + + root_bounds: tuple[int, int, int, int] + + def window_ids(self) -> list[int]: ... + + def window_record(self, window_id: int) -> X11WindowRecord | None: ... + + def capture_window(self, window_id: int, width: int, height: int) -> "Image.Image": ... + + +def _require_x11_session(environ: dict[str, str] | None = None) -> str: + """Return DISPLAY or refuse a session whose coordinate contract is unsafe.""" + values = os.environ if environ is None else environ + session_type = values.get("XDG_SESSION_TYPE", "").strip().casefold() + if session_type == "wayland" or values.get("WAYLAND_DISPLAY", "").strip(): + raise LinuxWindowCaptureError( + "window-scoped capture refuses native Wayland sessions: no portal " + "contract currently binds a selected window, exact pixels, and " + "event-time coordinates; use an X11 session" + ) + display = values.get("DISPLAY", "").strip() + if not display: + raise LinuxWindowCaptureError( + "window-scoped capture requires an X11 desktop and DISPLAY is not set" + ) + return display + + +def _bytes(value) -> bytes: + """Copy an xcffib byte list without retaining its reply buffer.""" + try: + return bytes(value) + except TypeError: + return b"".join(value) + + +def _u32_values(payload: bytes) -> list[int]: + """Decode X11 32-bit property items in native client byte order.""" + import struct + + if len(payload) % 4: + return [] + if not payload: + return [] + return list(struct.unpack(f"={len(payload) // 4}I", payload)) + + +def _rect_within( + inner: tuple[int, int, int, int], + outer: tuple[int, int, int, int], +) -> bool: + """Return whether a positive rectangle is fully inside another rectangle.""" + x, y, width, height = inner + ox, oy, outer_width, outer_height = outer + return ( + width > 0 + and height > 0 + and x >= ox + and y >= oy + and x + width <= ox + outer_width + and y + height <= oy + outer_height + ) + + +class X11CompositeClient(AbstractContextManager["X11CompositeClient"]): + """One short-lived checked XCB connection to the active X11 screen.""" + + def __init__(self, display: str | None = None) -> None: + display_name = display or _require_x11_session() + try: + import xcffib + import xcffib.composite + import xcffib.xproto + except (ImportError, OSError) as exc: + raise LinuxWindowCaptureError( + "Linux window capture requires xcffib and the system libxcb runtime" + ) from exc + + self._xcffib = xcffib + self._composite_module = xcffib.composite + self._xproto = xcffib.xproto + try: + self._conn = xcffib.connect(display=display_name) + except Exception as exc: + raise LinuxWindowCaptureError( + f"could not open X11 display {display_name!r}; authorize the capture process" + ) from exc + setup = self._conn.get_setup() + try: + self._screen = setup.roots[self._conn.pref_screen] + except (IndexError, TypeError) as exc: + self._conn.disconnect() + raise LinuxWindowCaptureError("X11 did not expose the selected screen") from exc + self._setup = setup + self._root = int(self._screen.root) + self.root_bounds = ( + 0, + 0, + int(self._screen.width_in_pixels), + int(self._screen.height_in_pixels), + ) + self._atoms: dict[str, int] = {} + + def __exit__(self, exc_type, exc, traceback) -> None: + self._conn.disconnect() + + def _atom(self, name: str, *, existing: bool = False) -> int: + key = f"{int(existing)}:{name}" + if key not in self._atoms: + reply = self._conn.core.InternAtom( + existing, + len(name.encode("ascii")), + name.encode("ascii"), + ).reply() + self._atoms[key] = int(reply.atom) + return self._atoms[key] + + def _property(self, window_id: int, name: str, *, length: int) -> tuple[int, bytes]: + atom = self._atom(name, existing=True) + if atom == 0: + return (0, b"") + reply = self._conn.core.GetProperty( + False, + window_id, + atom, + self._xproto.Atom.Any, + 0, + length, + ).reply() + return (int(reply.format), _bytes(reply.value)) + + def window_ids(self) -> list[int]: + """Return EWMH client windows in bottom-to-top stacking order.""" + for name in ("_NET_CLIENT_LIST_STACKING", "_NET_CLIENT_LIST"): + value_format, payload = self._property(self._root, name, length=1 << 20) + if value_format == 32: + window_ids = [value for value in _u32_values(payload) if value] + if window_ids: + return window_ids + raise LinuxWindowCaptureError( + "the X11 window manager does not expose an EWMH client window list" + ) + + def _title(self, window_id: int) -> str: + for name in ("_NET_WM_NAME", "WM_NAME"): + value_format, payload = self._property(window_id, name, length=4096) + if value_format == 8 and payload: + return payload.rstrip(b"\0").decode("utf-8", errors="replace") + return "" + + def _pid(self, window_id: int) -> int: + value_format, payload = self._property(window_id, "_NET_WM_PID", length=1) + values = _u32_values(payload) if value_format == 32 else [] + return values[0] if len(values) == 1 else 0 + + def window_record(self, window_id: int) -> X11WindowRecord | None: + """Read one viewability, identity, and root-pixel geometry snapshot.""" + try: + attributes = self._conn.core.GetWindowAttributes(window_id).reply() + if int(attributes._class) != self._xproto.WindowClass.InputOutput or bool( + attributes.override_redirect + ): + return None + geometry = self._conn.core.GetGeometry(window_id).reply() + translated = self._conn.core.TranslateCoordinates( + window_id, + self._root, + 0, + 0, + ).reply() + title = self._title(window_id) + pid = self._pid(window_id) + except Exception: + # A client can close between EWMH list and metadata requests. + return None + if not translated.same_screen: + return None + bounds = ( + int(translated.dst_x), + int(translated.dst_y), + int(geometry.width), + int(geometry.height), + ) + return X11WindowRecord( + window_id=window_id, + title=title, + pid=pid, + bounds=bounds, + viewable=(int(attributes.map_state) == self._xproto.MapState.Viewable), + ) + + def _visual(self, visual_id: int): + for depth in self._screen.allowed_depths: + for visual in depth.visuals: + if int(visual.visual_id) == visual_id: + return visual + raise LinuxWindowCaptureError(f"X11 visual {visual_id} is absent from the selected screen") + + def _pixel_format(self, depth: int, visual_id: int) -> X11PixelFormat: + formats = [value for value in self._setup.pixmap_formats if int(value.depth) == depth] + if len(formats) != 1: + raise LinuxWindowCaptureError(f"X11 did not expose one pixmap format for depth {depth}") + value = formats[0] + visual = self._visual(visual_id) + color_mask = int(visual.red_mask) | int(visual.green_mask) | int(visual.blue_mask) + alpha_mask = ((1 << depth) - 1) & ~color_mask + return X11PixelFormat( + bits_per_pixel=int(value.bits_per_pixel), + scanline_pad=int(value.scanline_pad), + image_byte_order=int(self._setup.image_byte_order), + red_mask=int(visual.red_mask), + green_mask=int(visual.green_mask), + blue_mask=int(visual.blue_mask), + alpha_mask=alpha_mask, + visual_class=int(visual._class), + ) + + def _name_window_pixmap(self, window_id: int) -> tuple[int, bool]: + """Name an existing redirect, or create one automatic redirect.""" + composite = self._conn(self._composite_module.key) + try: + version = composite.QueryVersion(0, 4).reply() + except Exception as exc: + raise LinuxWindowCaptureError( + "the X11 server does not expose the Composite extension" + ) from exc + if (int(version.major_version), int(version.minor_version)) < (0, 2): + raise LinuxWindowCaptureError("XComposite 0.2 or newer is required") + + pixmap = self._conn.generate_id() + try: + composite.NameWindowPixmap(window_id, pixmap, is_checked=True).check() + return (pixmap, False) + except Exception: + # A compositor usually redirects top-level windows already. A + # plain X server does not, so request automatic redirection and + # undo only the redirect owned by this connection. + redirected = False + try: + composite.RedirectWindow( + window_id, + self._composite_module.Redirect.Automatic, + is_checked=True, + ).check() + redirected = True + pixmap = self._conn.generate_id() + composite.NameWindowPixmap( + window_id, + pixmap, + is_checked=True, + ).check() + except Exception as exc: + if redirected: + try: + composite.UnredirectWindow( + window_id, + self._composite_module.Redirect.Automatic, + is_checked=True, + ).check() + except Exception: + pass + raise LinuxWindowCaptureError( + f"XComposite could not name window {window_id}'s backing pixmap" + ) from exc + return (pixmap, True) + + def capture_window(self, window_id: int, width: int, height: int) -> "Image.Image": + """Read the exact named-window pixmap, independent of root occlusion.""" + if width <= 0 or height <= 0: + raise LinuxWindowCaptureError(f"window {window_id} has empty bounds") + try: + attributes = self._conn.core.GetWindowAttributes(window_id).reply() + if int(attributes.map_state) != self._xproto.MapState.Viewable: + raise LinuxWindowCaptureError(f"window {window_id} is not viewable") + window_geometry = self._conn.core.GetGeometry(window_id).reply() + except LinuxWindowCaptureError: + raise + except Exception as exc: + raise LinuxWindowCaptureError( + f"could not revalidate X11 window {window_id} before capture" + ) from exc + actual_size = (int(window_geometry.width), int(window_geometry.height)) + if actual_size != (width, height): + raise LinuxWindowCaptureError( + f"window {window_id} changed size before capture: " + f"expected {(width, height)}, got {actual_size}" + ) + + pixmap = 0 + redirected = False + try: + pixmap, redirected = self._name_window_pixmap(window_id) + pixmap_geometry = self._conn.core.GetGeometry(pixmap).reply() + pixmap_size = (int(pixmap_geometry.width), int(pixmap_geometry.height)) + if pixmap_size != (width, height): + raise LinuxWindowCaptureError( + f"XComposite pixmap size {pixmap_size} does not match " + f"window size {(width, height)}" + ) + reply = self._conn.core.GetImage( + self._xproto.ImageFormat.ZPixmap, + pixmap, + 0, + 0, + width, + height, + 0xFFFFFFFF, + ).reply() + if int(reply.depth) != int(pixmap_geometry.depth): + raise LinuxWindowCaptureError("X11 image depth changed during capture") + pixel_format = self._pixel_format( + int(pixmap_geometry.depth), + int(attributes.visual), + ) + return decode_zpixmap(_bytes(reply.data), width, height, pixel_format) + except LinuxWindowCaptureError: + raise + except Exception as exc: + raise LinuxWindowCaptureError( + f"XComposite pixel capture failed for window {window_id}" + ) from exc + finally: + if pixmap: + try: + self._conn.core.FreePixmap(pixmap, is_checked=True).check() + except Exception: + pass + if redirected: + try: + composite = self._conn(self._composite_module.key) + composite.UnredirectWindow( + window_id, + self._composite_module.Redirect.Automatic, + is_checked=True, + ).check() + except Exception: + pass + + +def _channel(pixels, mask: int): + """Scale one contiguous TrueColor mask to an unsigned 8-bit channel.""" + import numpy as np + + if mask <= 0: + raise LinuxWindowCaptureError("X11 TrueColor channel mask is empty") + shift = (mask & -mask).bit_length() - 1 + maximum = mask >> shift + if maximum & (maximum + 1): + raise LinuxWindowCaptureError("X11 TrueColor channel mask is not contiguous") + values = ((pixels & mask) >> shift).astype(np.uint64) + return ((values * 255 + maximum // 2) // maximum).astype(np.uint8) + + +def decode_zpixmap( + payload: bytes, + width: int, + height: int, + pixel_format: X11PixelFormat, +) -> "Image.Image": + """Decode an X11 TrueColor ZPixmap without assuming BGRA or 32-bit rows.""" + import numpy as np + from PIL import Image + + if width <= 0 or height <= 0: + raise LinuxWindowCaptureError("X11 image dimensions must be positive") + if pixel_format.visual_class != 4: + raise LinuxWindowCaptureError( + "X11 window capture supports TrueColor visuals only; indexed colormaps " + "cannot be decoded without a live colormap snapshot" + ) + bits_per_pixel = pixel_format.bits_per_pixel + scanline_pad = pixel_format.scanline_pad + if bits_per_pixel not in {16, 24, 32}: + raise LinuxWindowCaptureError( + f"unsupported X11 ZPixmap depth: {bits_per_pixel} bits per pixel" + ) + if scanline_pad not in {8, 16, 32}: + raise LinuxWindowCaptureError(f"unsupported X11 scanline padding: {scanline_pad} bits") + row_bits = width * bits_per_pixel + row_bytes = ((row_bits + scanline_pad - 1) // scanline_pad) * (scanline_pad // 8) + required = row_bytes * height + if len(payload) < required: + raise LinuxWindowCaptureError( + f"X11 image payload is truncated: expected {required} bytes, got {len(payload)}" + ) + raw = np.frombuffer(payload, dtype=np.uint8, count=required) + little_endian = pixel_format.image_byte_order == 0 + if bits_per_pixel == 16: + dtype = np.dtype("u2") + pixels = np.ndarray( + shape=(height, width), + dtype=dtype, + buffer=raw, + strides=(row_bytes, 2), + ).astype(np.uint32) + elif bits_per_pixel == 32: + dtype = np.dtype("u4") + pixels = np.ndarray( + shape=(height, width), + dtype=dtype, + buffer=raw, + strides=(row_bytes, 4), + ).astype(np.uint32) + else: + octets = np.ndarray( + shape=(height, width, 3), + dtype=np.uint8, + buffer=raw, + strides=(row_bytes, 3, 1), + ).astype(np.uint32) + if little_endian: + pixels = octets[..., 0] | (octets[..., 1] << 8) | (octets[..., 2] << 16) + else: + pixels = (octets[..., 0] << 16) | (octets[..., 1] << 8) | octets[..., 2] + if pixel_format.alpha_mask: + alpha = _channel(pixels, pixel_format.alpha_mask) + if not bool(np.all(alpha == 255)): + raise LinuxWindowCaptureError( + "X11 window pixels contain transparency and cannot be flattened into " + "exact RGB pixels without the compositor's retained background" + ) + rgb = np.stack( + ( + _channel(pixels, pixel_format.red_mask), + _channel(pixels, pixel_format.green_mask), + _channel(pixels, pixel_format.blue_mask), + ), + axis=-1, + ) + return Image.fromarray(rgb) + + +def _default_process_identity(pid: int) -> tuple[str, float]: + """Bind an EWMH PID to the exact live process instance.""" + import psutil + + process = psutil.Process(pid) + return (process.name(), float(process.create_time())) + + +def resolve_window_linux( + target: "WindowTarget", + *, + client_factory: Callable[[], AbstractContextManager[_X11ClientProtocol]] | None = None, + process_identity: Callable[[int], tuple[str, float]] | None = None, +) -> "TargetWindow | None": + """Resolve the largest matching, fully on-screen EWMH client window.""" + _require_x11_session() + from openadapt_capture.window_capture import TargetWindow + + factory = client_factory or X11CompositeClient + identity_reader = process_identity or _default_process_identity + owner_text = target.owner.casefold() if target.owner else None + title_text = target.title.casefold() if target.title else None + matches: list[tuple[int, TargetWindow]] = [] + try: + with factory() as client: + root_bounds = client.root_bounds + for stacking_index, window_id in enumerate(client.window_ids()): + record = client.window_record(window_id) + if ( + record is None + or not record.viewable + or record.pid <= 0 + or not _rect_within(record.bounds, root_bounds) + ): + continue + if title_text is not None and title_text not in record.title.casefold(): + continue + try: + owner, process_start_time = identity_reader(record.pid) + except Exception: + continue + if not owner or owner_text is not None and owner_text not in owner.casefold(): + continue + matches.append( + ( + stacking_index, + TargetWindow( + window_id=record.window_id, + owner=owner, + title=record.title, + pid=record.pid, + bounds=tuple(float(value) for value in record.bounds), + on_screen=True, + process_start_time=process_start_time, + coordinate_source="x11-root-physical-pixels", + ), + ) + ) + except LinuxWindowCaptureError: + raise + except Exception as exc: + raise LinuxWindowCaptureError("X11 window resolution failed") from exc + if not matches: + return None + return max( + matches, + key=lambda item: ( + item[1].bounds[2] * item[1].bounds[3], + item[0], + ), + )[1] + + +def capture_window_linux( + window: "TargetWindow", + *, + client_factory: Callable[[], AbstractContextManager[_X11ClientProtocol]] | None = None, +) -> "Image.Image": + """Capture one process-bound X11 window from its named backing pixmap.""" + _require_x11_session() + if not window.on_screen: + raise LinuxWindowCaptureError(f"window {window.window_id} is not on screen") + x, y, width_value, height_value = window.bounds + values = (x, y, width_value, height_value) + if any(not float(value).is_integer() for value in values): + raise LinuxWindowCaptureError( + "X11 root-pixel window bounds must contain integer coordinates" + ) + width = int(width_value) + height = int(height_value) + factory = client_factory or X11CompositeClient + try: + with factory() as client: + if not _rect_within(tuple(int(value) for value in values), client.root_bounds): + raise LinuxWindowCaptureError( + f"window {window.window_id} is not fully on the X11 root screen" + ) + image = client.capture_window(window.window_id, width, height) + except LinuxWindowCaptureError: + raise + except Exception as exc: + raise LinuxWindowCaptureError( + f"XComposite pixel capture failed for window {window.window_id}" + ) from exc + if image.size != (width, height): + raise LinuxWindowCaptureError( + f"XComposite returned {image.size} for window bounds {(width, height)}" + ) + return image + + +__all__ = [ + "LinuxWindowCaptureError", + "X11CompositeClient", + "X11PixelFormat", + "X11WindowRecord", + "capture_window_linux", + "decode_zpixmap", + "resolve_window_linux", +] diff --git a/pyproject.toml b/pyproject.toml index eeaafd5..3a45b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ dependencies = [ # Native Windows structural observation. Platform-gated so non-Windows # installs and imports do not load the Windows UIA stack. "pywinauto>=0.6.9; sys_platform == 'win32'", + # Checked X11 metadata and XComposite named-window pixels on Linux. + "xcffib>=1.8.0; sys_platform == 'linux'", ] [project.optional-dependencies] diff --git a/tests/test_window_capture_linux.py b/tests/test_window_capture_linux.py new file mode 100644 index 0000000..efb1552 --- /dev/null +++ b/tests/test_window_capture_linux.py @@ -0,0 +1,446 @@ +"""Linux X11 window producer contracts, including an opt-in live check.""" + +from __future__ import annotations + +import os +import sys +from types import SimpleNamespace + +import pytest +from PIL import Image + +from openadapt_capture.desktop_capture import DesktopCaptureScope +from openadapt_capture.window_capture import ( + TargetWindow, + WindowCaptureError, + WindowCaptureScope, + WindowTarget, +) +from openadapt_capture.window_capture_linux import ( + LinuxWindowCaptureError, + X11CompositeClient, + X11PixelFormat, + X11WindowRecord, + capture_window_linux, + decode_zpixmap, + resolve_window_linux, +) + + +class FakeX11Client: + """Display-free implementation of the Linux producer's X11 seam.""" + + def __init__( + self, + records: list[X11WindowRecord], + *, + root_bounds: tuple[int, int, int, int] = (0, 0, 1920, 1080), + image: Image.Image | None = None, + ) -> None: + self.records = {record.window_id: record for record in records} + self.order = [record.window_id for record in records] + self.root_bounds = root_bounds + self.image = image + self.capture_calls: list[tuple[int, int, int]] = [] + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def window_ids(self) -> list[int]: + return list(self.order) + + def window_record(self, window_id: int) -> X11WindowRecord | None: + return self.records.get(window_id) + + def capture_window(self, window_id: int, width: int, height: int) -> Image.Image: + self.capture_calls.append((window_id, width, height)) + return self.image or Image.new("RGB", (width, height), "navy") + + +@pytest.fixture +def x11_environment(monkeypatch): + monkeypatch.setenv("DISPLAY", ":99") + monkeypatch.setenv("XDG_SESSION_TYPE", "x11") + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + + +def _record( + window_id: int, + *, + title: str = "Remote session", + pid: int = 1000, + bounds: tuple[int, int, int, int] = (100, 80, 800, 600), + viewable: bool = True, +) -> X11WindowRecord: + return X11WindowRecord( + window_id=window_id, + title=title, + pid=pid, + bounds=bounds, + viewable=viewable, + ) + + +def test_resolver_binds_process_and_root_pixel_coordinates(x11_environment) -> None: + client = FakeX11Client([_record(41), _record(42, bounds=(50, 40, 1200, 700))]) + identities = { + 1000: ("citrix-workspace", 1700000000.25), + } + + window = resolve_window_linux( + WindowTarget(owner="citrix", title="remote"), + client_factory=lambda: client, + process_identity=identities.__getitem__, + ) + + assert window == TargetWindow( + window_id=42, + owner="citrix-workspace", + title="Remote session", + pid=1000, + bounds=(50.0, 40.0, 1200.0, 700.0), + on_screen=True, + process_start_time=1700000000.25, + coordinate_source="x11-root-physical-pixels", + ) + + +@pytest.mark.parametrize( + "record", + [ + _record(1, viewable=False), + _record(1, bounds=(-1, 0, 100, 100)), + _record(1, bounds=(1850, 1000, 100, 100)), + _record(1, pid=0), + ], +) +def test_resolver_refuses_unviewable_offscreen_or_unbound_windows( + x11_environment, + record: X11WindowRecord, +) -> None: + client = FakeX11Client([record]) + assert ( + resolve_window_linux( + WindowTarget(owner="citrix"), + client_factory=lambda: client, + process_identity=lambda _pid: ("citrix", 100.0), + ) + is None + ) + + +def test_resolver_refuses_process_lookup_failure(x11_environment) -> None: + client = FakeX11Client([_record(1)]) + + def missing_process(_pid: int): + raise ProcessLookupError + + assert ( + resolve_window_linux( + WindowTarget(title="remote"), + client_factory=lambda: client, + process_identity=missing_process, + ) + is None + ) + + +@pytest.mark.parametrize( + ("session_type", "wayland_display"), + [("wayland", None), ("x11", "wayland-0")], +) +def test_resolver_refuses_native_wayland( + monkeypatch, + session_type: str, + wayland_display: str | None, +) -> None: + monkeypatch.setenv("DISPLAY", ":0") + monkeypatch.setenv("XDG_SESSION_TYPE", session_type) + if wayland_display is None: + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + else: + monkeypatch.setenv("WAYLAND_DISPLAY", wayland_display) + + with pytest.raises(LinuxWindowCaptureError, match="refuses native Wayland"): + resolve_window_linux( + WindowTarget(owner="citrix"), + client_factory=lambda: FakeX11Client([]), + ) + + +def test_resolver_refuses_missing_x11_display(monkeypatch) -> None: + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + monkeypatch.delenv("XDG_SESSION_TYPE", raising=False) + + with pytest.raises(LinuxWindowCaptureError, match="DISPLAY is not set"): + resolve_window_linux( + WindowTarget(owner="citrix"), + client_factory=lambda: FakeX11Client([]), + ) + + +def test_capture_uses_exact_named_window_size(x11_environment) -> None: + expected = Image.new("RGB", (800, 600), "purple") + client = FakeX11Client([_record(42)], image=expected) + window = TargetWindow( + window_id=42, + owner="citrix", + title="Remote session", + pid=1000, + bounds=(100.0, 80.0, 800.0, 600.0), + process_start_time=100.0, + coordinate_source="x11-root-physical-pixels", + ) + + image = capture_window_linux(window, client_factory=lambda: client) + + assert image is expected + assert client.capture_calls == [(42, 800, 600)] + + +def test_capture_refuses_wrong_pixel_extent(x11_environment) -> None: + client = FakeX11Client([_record(42)], image=Image.new("RGB", (799, 600))) + window = TargetWindow( + window_id=42, + owner="citrix", + title="Remote session", + pid=1000, + bounds=(100.0, 80.0, 800.0, 600.0), + process_start_time=100.0, + coordinate_source="x11-root-physical-pixels", + ) + with pytest.raises(LinuxWindowCaptureError, match="returned .* for window bounds"): + capture_window_linux(window, client_factory=lambda: client) + + +class _Cookie: + def __init__(self, *, reply=None, error: Exception | None = None) -> None: + self._reply = reply + self._error = error + + def reply(self): + if self._error: + raise self._error + return self._reply + + def check(self) -> None: + if self._error: + raise self._error + + +class _CompositeFailure: + def __init__(self) -> None: + self.name_calls = 0 + self.redirect_calls = 0 + self.unredirect_calls = 0 + + def QueryVersion(self, *_args): + return _Cookie(reply=SimpleNamespace(major_version=0, minor_version=4)) + + def NameWindowPixmap(self, *_args, **_kwargs): + self.name_calls += 1 + return _Cookie(error=RuntimeError("name failed")) + + def RedirectWindow(self, *_args, **_kwargs): + self.redirect_calls += 1 + return _Cookie() + + def UnredirectWindow(self, *_args, **_kwargs): + self.unredirect_calls += 1 + return _Cookie() + + +def test_xcomposite_redirect_is_undone_when_pixmap_naming_fails() -> None: + composite = _CompositeFailure() + client = object.__new__(X11CompositeClient) + client._composite_module = SimpleNamespace( + key="composite", Redirect=SimpleNamespace(Automatic=0) + ) + # Special methods resolve on the type, so use a small callable connection. + client._conn = type( + "FakeConnection", + (), + { + "generate_id": lambda self: next(self.ids), + "__call__": lambda self, _key: composite, + "ids": iter([100, 101]), + }, + )() + + with pytest.raises(LinuxWindowCaptureError, match="could not name"): + client._name_window_pixmap(42) + + assert composite.name_calls == 2 + assert composite.redirect_calls == 1 + assert composite.unredirect_calls == 1 + + +def test_decode_32_bit_little_endian_truecolor() -> None: + image = decode_zpixmap( + bytes( + [ + 0x00, + 0x00, + 0xFF, + 0x00, + 0x00, + 0xFF, + 0x00, + 0x00, + ] + ), + 2, + 1, + X11PixelFormat( + bits_per_pixel=32, + scanline_pad=32, + image_byte_order=0, + red_mask=0x00FF0000, + green_mask=0x0000FF00, + blue_mask=0x000000FF, + ), + ) + assert [image.getpixel((x, 0)) for x in range(2)] == [(255, 0, 0), (0, 255, 0)] + + +def test_decode_16_bit_rgb565_with_row_padding() -> None: + # One RGB565 pixel plus a two-byte scanline pad. + image = decode_zpixmap( + bytes([0x00, 0xF8, 0x00, 0x00]), + 1, + 1, + X11PixelFormat( + bits_per_pixel=16, + scanline_pad=32, + image_byte_order=0, + red_mask=0xF800, + green_mask=0x07E0, + blue_mask=0x001F, + ), + ) + assert image.getpixel((0, 0)) == (255, 0, 0) + + +def test_decode_refuses_indexed_visual() -> None: + with pytest.raises(LinuxWindowCaptureError, match="TrueColor visuals only"): + decode_zpixmap( + b"\0" * 4, + 1, + 1, + X11PixelFormat( + bits_per_pixel=32, + scanline_pad=32, + image_byte_order=0, + red_mask=0, + green_mask=0, + blue_mask=0, + visual_class=3, + ), + ) + + +def test_decode_refuses_nonopaque_argb_pixels() -> None: + with pytest.raises(LinuxWindowCaptureError, match="contain transparency"): + decode_zpixmap( + bytes([0x00, 0x00, 0xFF, 0x7F]), + 1, + 1, + X11PixelFormat( + bits_per_pixel=32, + scanline_pad=32, + image_byte_order=0, + red_mask=0x00FF0000, + green_mask=0x0000FF00, + blue_mask=0x000000FF, + alpha_mask=0xFF000000, + ), + ) + + +def _linux_target(bounds=(700.0, 100.0, 600.0, 500.0)) -> TargetWindow: + return TargetWindow( + window_id=42, + owner="citrix", + title="Remote session", + pid=1000, + bounds=bounds, + process_start_time=100.0, + coordinate_source="x11-root-physical-pixels", + ) + + +def _topology(monitors: list[list[int]]) -> dict: + return { + "schema_version": "openadapt.capture.display-topology/v1", + "coordinate_space": "virtual_desktop_pixels", + "monitors": monitors, + "topology_sha256": "a" * 64, + } + + +def test_scope_accepts_window_spanning_adjacent_monitors() -> None: + target = _linux_target() + scope = WindowCaptureScope( + WindowTarget(owner="citrix"), + resolver=lambda _target: target, + capturer=lambda _window: Image.new("RGB", (600, 500)), + ) + scope.bind_display_topology( + _topology([[0, 0, 1000, 800], [1000, 0, 1000, 800]]), + lambda **_kwargs: None, + ) + image, _changed = scope.capture_frame() + assert image.size == (600, 500) + + +def test_scope_refuses_window_crossing_a_topology_gap() -> None: + target = _linux_target() + scope = WindowCaptureScope( + WindowTarget(owner="citrix"), + resolver=lambda _target: target, + capturer=lambda _window: Image.new("RGB", (600, 500)), + ) + scope.bind_display_topology( + _topology([[0, 0, 800, 800], [1000, 0, 1000, 800]]), + lambda **_kwargs: None, + ) + with pytest.raises(WindowCaptureError, match="not fully covered"): + scope.capture_frame() + + +_LIVE_LINUX = sys.platform.startswith("linux") +_LIVE_ENABLED = os.environ.get("OPENADAPT_CAPTURE_LINUX_WINDOW_QUALIFICATION") == "1" + + +@pytest.mark.slow +@pytest.mark.skipif(not _LIVE_LINUX, reason="Linux X11 qualification runs only on Linux") +@pytest.mark.skipif( + not _LIVE_ENABLED, + reason="set OPENADAPT_CAPTURE_LINUX_WINDOW_QUALIFICATION=1 on an interactive X11 rig", +) +def test_live_linux_xcomposite_window_capture() -> None: + """Qualify identity, topology, exact pixels, and revalidation on a live rig.""" + owner = os.environ.get("OPENADAPT_WINDOW_SMOKE_OWNER", "").strip() + title = os.environ.get("OPENADAPT_WINDOW_SMOKE_TITLE", "").strip() or None + assert owner, "OPENADAPT_WINDOW_SMOKE_OWNER must name the qualification application" + scope = WindowCaptureScope(WindowTarget(owner=owner, title=title)) + desktop = DesktopCaptureScope.current() + scope.bind_display_topology(desktop.snapshot(), desktop.assert_current) + + first, changed = scope.capture_frame() + assert changed is True + state = scope.window_event_data()["state"] + assert state["coordinate_source"] == "x11-root-physical-pixels" + assert state["source_viewport"] == [int(state["bounds"][2]), int(state["bounds"][3])] + assert first.size == tuple(state["source_viewport"]) + assert state["pid"] > 0 + assert state["process_start_time"] > 0 + + second, _changed = scope.capture_frame() + scope.assert_current() + assert second.size == first.size + assert scope.snapshot()["window_id"] == state["window_id"] From 50f7423dd58217fee8738a9ef2af1374bae86984 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 17:19:26 -0400 Subject: [PATCH 5/6] fix: preserve native frame causality --- openadapt_capture/capture.py | 76 +++++++--- openadapt_capture/recorder.py | 86 ++++++----- tests/test_capture_terminal.py | 255 ++++++++++++++++++++++++++++++++- tests/test_window_capture.py | 20 ++- 4 files changed, 363 insertions(+), 74 deletions(-) diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 52244fe..aac934f 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -5,6 +5,7 @@ from __future__ import annotations +import hashlib import io from dataclasses import dataclass from pathlib import Path @@ -571,9 +572,7 @@ def _validate_database_contract( calculated_last = max(all_ordinals, default=None) if calculated_last != last_source_ordinal: - raise InvalidCaptureEvent( - "sealed last source ordinal differs from the immutable database" - ) + raise InvalidCaptureEvent("sealed last source ordinal differs from the immutable database") for event in rows_by_kind["action"]: _convert_action_event(event) @@ -602,9 +601,7 @@ def _validate_database_contract( if is_v2: window_events = capture.window_capture_events_v2() windows_by_ordinal = {event.source_ordinal: event for event in window_events} - screenshots_by_ordinal = { - row.source_ordinal: row for row in rows_by_kind["screen"] - } + screenshots_by_ordinal = {row.source_ordinal: row for row in rows_by_kind["screen"]} if set(windows_by_ordinal) != set(screenshots_by_ordinal): raise InvalidCaptureEvent("sealed v2 frame/window pairs are not a bijection") allowed_pair_ordinals = set(windows_by_ordinal) @@ -622,6 +619,11 @@ def _validate_database_contract( if screenshot.png_data: from PIL import Image + retained_sha256 = hashlib.sha256(screenshot.png_data).hexdigest() + if screenshot.png_sha256 != retained_sha256: + raise InvalidCaptureEvent( + "sealed v2 frame PNG digest differs from its retained bytes" + ) try: with Image.open(io.BytesIO(screenshot.png_data)) as retained: retained.load() @@ -632,6 +634,10 @@ def _validate_database_contract( raise InvalidCaptureEvent( "sealed v2 frame dimensions differ from its geometry viewport" ) + elif screenshot.png_sha256 is not None: + raise InvalidCaptureEvent( + "sealed v2 frame has a PNG digest without retained PNG bytes" + ) current_identity = ( state.window_id, state.pid, @@ -657,16 +663,20 @@ def _validate_database_contract( for action in rows_by_kind["action"]: if ( action.window_geometry_generation is None - or action.screenshot_source_ordinal - != action.window_event_source_ordinal + or action.screenshot_source_ordinal != action.window_event_source_ordinal ): raise InvalidCaptureEvent("sealed v2 action has an incomplete frame binding") bound = windows_by_ordinal.get(action.window_event_source_ordinal) if bound is None or ( - bound.window_capture_v2.geometry_generation - != action.window_geometry_generation + bound.window_capture_v2.geometry_generation != action.window_geometry_generation ): raise InvalidCaptureEvent("sealed v2 action names the wrong geometry epoch") + if not any( + frame_ordinal > action.source_ordinal for frame_ordinal in allowed_pair_ordinals + ): + raise InvalidCaptureEvent( + "sealed v2 action has no ordinal-later retained after frame" + ) owners: dict[int, set[str]] = {} for kind, ordinals in ordinals_by_kind.items(): @@ -677,6 +687,8 @@ def _validate_database_contract( ordinal in allowed_pair_ordinals and kinds == {"screen", "window"} ): raise InvalidCaptureEvent("sealed journal reuses a source ordinal across events") + if is_v2 and sorted(owners) != list(range(1, (last_source_ordinal or 0) + 1)): + raise InvalidCaptureEvent("sealed v2 source journal has a missing ordinal") expected_video_count = event_counts.get("video") if not isinstance(expected_video_count, int) or expected_video_count < 0: @@ -692,6 +704,27 @@ def _validate_database_contract( raise InvalidCaptureEvent("sealed MP4 has no source-ordinal frame bindings") if len(timing[3]) != expected_video_count: raise InvalidCaptureEvent("sealed video count differs from its MP4 bindings") + if is_v2: + video_source_ordinals = [source_ordinal for _, source_ordinal in timing[3]] + expected_source_ordinals = sorted(ordinals_by_kind["screen"]) + if video_source_ordinals != expected_source_ordinals: + raise InvalidCaptureEvent( + "sealed v2 MP4 source bindings differ from retained frame ordinals" + ) + capture_bindings = timing[2] + if capture_bindings is None: + raise InvalidCaptureEvent("sealed v2 MP4 has no exact capture-time frame bindings") + capture_by_index = dict(capture_bindings) + screenshots_by_ordinal = { + int(row.source_ordinal): row for row in rows_by_kind["screen"] + } + for encoded_index, source_ordinal in timing[3]: + screenshot = screenshots_by_ordinal[source_ordinal] + if capture_by_index.get(encoded_index) != screenshot.timestamp: + raise InvalidCaptureEvent( + "sealed v2 MP4 source and capture-time bindings differ " + "from the retained frame" + ) elif capture.video_path is not None: raise InvalidCaptureEvent("sealed capture inventories an MP4 but claims no video frames") @@ -865,13 +898,16 @@ def task_description(self) -> str | None: @property def video_path(self) -> Path | None: - """Path to video file if exists.""" - # Legacy format: oa_recording-{timestamp}.mp4 - for p in self.capture_dir.glob("oa_recording-*.mp4"): - return p - # Fallback: video.mp4 - video_path = self.capture_dir / "video.mp4" - return video_path if video_path.exists() else None + """Return the only recognized video file, or refuse ambiguity.""" + candidates = sorted(self.capture_dir.glob("oa_recording-*.mp4")) + fallback = self.capture_dir / "video.mp4" + if fallback.exists(): + candidates.append(fallback) + if len(candidates) > 1: + raise InvalidCaptureEvent( + "capture has multiple recognized MP4 artifacts; exact frame bindings are ambiguous" + ) + return candidates[0] if candidates else None @property def audio_path(self) -> Path | None: @@ -1163,11 +1199,7 @@ def get_exact_frame( return Image.open(io.BytesIO(screenshot.png_data)).convert("RGB") raise LookupError( f"no frame was retained at exactly {capture_timestamp!r}" - + ( - f" with source ordinal {source_ordinal} " - if source_ordinal is not None - else " " - ) + + (f" with source ordinal {source_ordinal} " if source_ordinal is not None else " ") + "(fail-closed; refusing a nearest-frame substitute)" ) diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 67d022f..94ad115 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -198,6 +198,12 @@ def reserve(self, timestamp: float) -> EventReservation: entry = self._reserve_locked(timestamp) return EventReservation(self, entry) + @property + def last_source_ordinal(self) -> int | None: + """Return the last ordinal reserved by any source observer.""" + with self._condition: + return self._next_sequence - 1 or None + def _reserve_locked(self, timestamp: float) -> _JournalEntry: entry = _JournalEntry(timestamp, self._next_sequence) self._next_sequence += 1 @@ -1176,9 +1182,7 @@ def trigger_action_event( ) else: reservation = ( - event_q.reserve(event_timestamp) - if isinstance(event_q, OrderedEventJournal) - else None + event_q.reserve(event_timestamp) if isinstance(event_q, OrderedEventJournal) else None ) try: if isinstance(coordinate_scope, WindowCaptureScope): @@ -1429,33 +1433,37 @@ def capture_one() -> tuple[float, float]: nonlocal started t_start = time.perf_counter() if window_scope is not None: - with window_scope.observation_boundary(): - # Any failed capture terminates the session. Retrying would omit a - # frame while input continues and could produce complete-looking - # evidence with a missing interval. - screenshot, _window_changed = window_scope.capture_frame(publish=False) - t_screenshot = time.perf_counter() - if screenshot is None: - raise WindowCaptureError("the captured screenshot was empty") - if not started: - started_event.set() - started = True - frame_timestamp = utils.get_timestamp() - if not isinstance(event_q, OrderedEventJournal): - raise WindowCaptureError( - "window-scoped capture requires the ordered event journal" - ) - generation = window_scope.current_generation() - scoped_frame = WindowScopedFrame( - image=screenshot, - window_event_data=window_scope.window_event_data(), - geometry_generation=generation, - ) - event_q.commit_window_frame( - Event(frame_timestamp, "screen", scoped_frame), - window_scope, - generation, - ) + # Do not hold the observation boundary during pixel acquisition. + # An OS input callback that arrives while the grab is in flight must + # reserve and bind the previously published frame before this new + # frame enters the journal. The frame can contain pixels rendered + # after that input, so publishing it first would make a post-action + # image look like the action's before evidence. + # + # Any failed capture terminates the session. Retrying would omit a + # frame while input continues and could produce complete-looking + # evidence with a missing interval. + screenshot, _window_changed = window_scope.capture_frame(publish=False) + t_screenshot = time.perf_counter() + if screenshot is None: + raise WindowCaptureError("the captured screenshot was empty") + if not started: + started_event.set() + started = True + frame_timestamp = utils.get_timestamp() + if not isinstance(event_q, OrderedEventJournal): + raise WindowCaptureError("window-scoped capture requires the ordered event journal") + generation = window_scope.current_generation() + scoped_frame = WindowScopedFrame( + image=screenshot, + window_event_data=window_scope.window_event_data(), + geometry_generation=generation, + ) + event_q.commit_window_frame( + Event(frame_timestamp, "screen", scoped_frame), + window_scope, + generation, + ) return t_start, t_screenshot if desktop_scope is not None: # A monitor can move or change scale while the combined frame keeps @@ -2016,7 +2024,7 @@ def record( window_owner: str | None = None, window_title: str | None = None, structural_observer: StructuralObserver | None = None, -) -> None: +) -> int | None: """Record native screenshots, action events, and window events. Args: @@ -2578,6 +2586,7 @@ def record( # TODO: consolidate terminate_recording and status_pipe if status_pipe: status_pipe.send({"type": "record.stopped"}) + return event_q.last_source_ordinal if window_scope is not None else None class Recorder: @@ -2696,6 +2705,7 @@ def __init__( self._record_thread: threading.Thread | None = None self._status_thread: threading.Thread | None = None self._capture = None # lazy CaptureSession + self._last_source_ordinal: int | None = None self._worker_error: BaseException | None = None self._worker_error_lock = threading.Lock() self._structural_observer = structural_observer @@ -2866,12 +2876,8 @@ def _verify_completed_capture(self) -> None: + list(capture._recording.window_events) + list(capture._recording.browser_events) ) - last_source_ordinal = max( - ( - row.source_ordinal - for row in database_rows - if row.source_ordinal is not None - ), + last_source_ordinal = self._last_source_ordinal or max( + (row.source_ordinal for row in database_rows if row.source_ordinal is not None), default=None, ) try: @@ -2900,7 +2906,7 @@ def _seal_completed_capture(self) -> None: db_path = Path(self.capture_dir) / "recording.db" database = sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True) try: - last_source_ordinal = max( + last_source_ordinal = self._last_source_ordinal or max( (database.execute(f"SELECT MAX(source_ordinal) FROM {table}").fetchone()[0] or 0) for table in ( "action_event", @@ -3004,7 +3010,7 @@ def _run_record(self) -> None: try: with config_override(self._recording_config): - record( + last_source_ordinal = record( task_description=self.task_description, capture_dir=self.capture_dir, terminate_processing=self._terminate_processing, @@ -3018,6 +3024,8 @@ def _run_record(self) -> None: send_profile=self._send_profile, structural_observer=self._structural_observer, ) + if last_source_ordinal is not None: + self._last_source_ordinal = last_source_ordinal self.check_health() if self._ready_event.is_set(): self._verify_completed_capture() diff --git a/tests/test_capture_terminal.py b/tests/test_capture_terminal.py index 66b4000..8c0d9ca 100644 --- a/tests/test_capture_terminal.py +++ b/tests/test_capture_terminal.py @@ -3,14 +3,18 @@ from __future__ import annotations import hashlib +import io import json from pathlib import Path import pytest +from PIL import Image import openadapt_capture.terminal as terminal_module from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud +from openadapt_capture.db.models import ActionEvent, Recording, Screenshot, WindowEvent +from openadapt_capture.events import window_geometry_epoch_sha256 from openadapt_capture.terminal import ( ARTIFACT_MANIFEST_FILENAME, CAPTURE_TERMINAL_FILENAME, @@ -62,6 +66,143 @@ def _seal(capture_dir: Path): ) +def _v2_capture_directory( + root: Path, + *, + frame_ordinals: tuple[int, ...] = (1, 3), + action_ordinal: int | None = 2, + wrong_png_digest: bool = False, + video: bool = False, +) -> Path: + """Build one sealed v2 capture for artifact-contract regressions.""" + capture_dir = root / "capture" + capture_dir.mkdir(parents=True) + state = { + "schema_version": "openadapt.capture.window-scoped/v2", + "window_capture": True, + "window_id": "42", + "owner": "FixtureApp", + "pid": 4242, + "process_start_time": 9.0, + "coordinate_source": "test-screen-points", + "geometry_generation": 1, + "display_topology_sha256": "a" * 64, + "bounds": [10.0, 20.0, 80.0, 60.0], + "scale": 1.0, + "scale_x": 1.0, + "scale_y": 1.0, + "viewport": [80, 60], + "source_viewport": [80, 60], + "content_rect": [0, 0, 80, 60], + "fit_scale": 1.0, + "on_screen": True, + } + state["geometry_epoch_sha256"] = window_geometry_epoch_sha256(state) + config = { + "capture_window": { + **state, + "target": {"owner": "FixtureApp", "title": None}, + "title": "Fixture Window", + "initial_bounds": state["bounds"], + "coordinate_space": "window_pixels", + } + } + + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + try: + recording = Recording( + timestamp=10.0, + monitor_width=80, + monitor_height=60, + platform="linux", + task_description="sealed v2 capture", + double_click_interval_seconds=0.5, + double_click_distance_pixels=5.0, + config=config, + ) + session.add(recording) + session.flush() + frames: dict[int, tuple[Screenshot, WindowEvent]] = {} + for index, ordinal in enumerate(frame_ordinals): + timestamp = 11.0 + index + output = io.BytesIO() + Image.new("RGB", (80, 60), (20 + index, 40, 60)).save( + output, + format="PNG", + ) + png = output.getvalue() + screenshot = Screenshot( + recording_id=recording.id, + recording_timestamp=10.0, + timestamp=timestamp, + source_ordinal=ordinal, + png_data=png, + png_sha256=("f" * 64 if wrong_png_digest else hashlib.sha256(png).hexdigest()), + ) + window = WindowEvent( + recording_id=recording.id, + recording_timestamp=10.0, + timestamp=timestamp, + source_ordinal=ordinal, + title="Fixture Window", + left=10, + top=20, + width=80, + height=60, + window_id="42", + state=state, + ) + session.add_all((screenshot, window)) + frames[ordinal] = (screenshot, window) + if action_ordinal is not None: + before_ordinal = frame_ordinals[0] + before, window = frames[before_ordinal] + session.add( + ActionEvent( + recording_id=recording.id, + recording_timestamp=10.0, + timestamp=11.5, + source_ordinal=action_ordinal, + name="click", + mouse_x=20.0, + mouse_y=20.0, + mouse_button_name="left", + mouse_pressed=False, + screenshot=before, + screenshot_timestamp=before.timestamp, + screenshot_source_ordinal=before_ordinal, + window_event=window, + window_event_timestamp=window.timestamp, + window_event_source_ordinal=before_ordinal, + window_geometry_generation=1, + ) + ) + session.commit() + finally: + session.close() + engine.dispose() + + if video: + (capture_dir / "video.mp4").write_bytes(b"fixture") + seal_capture( + capture_dir, + session_id="v2-session", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={ + "action": int(action_ordinal is not None), + "screen": len(frame_ordinals), + "window": len(frame_ordinals), + "browser": 0, + "video": len(frame_ordinals) if video else 0, + }, + last_source_ordinal=max((*frame_ordinals, action_ordinal or 0)), + ) + return capture_dir + + def test_terminal_binds_canonical_manifest_bytes_including_newline(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) terminal = _seal(capture_dir) @@ -69,9 +210,10 @@ def test_terminal_binds_canonical_manifest_bytes_including_newline(tmp_path) -> assert manifest_raw.endswith(b"\n") assert terminal.artifact_manifest_size_bytes == len(manifest_raw) - assert terminal.artifact_manifest_sha256 == hashlib.sha256( - b"openadapt.capture-artifact-manifest.v1\0" + manifest_raw - ).hexdigest() + assert ( + terminal.artifact_manifest_sha256 + == hashlib.sha256(b"openadapt.capture-artifact-manifest.v1\0" + manifest_raw).hexdigest() + ) assert (capture_dir / CAPTURE_TERMINAL_FILENAME).read_bytes().endswith(b"\n") verified_terminal, manifest = verify_capture_artifacts(capture_dir) assert verified_terminal == terminal @@ -153,9 +295,12 @@ def test_verified_loader_uses_a_private_snapshot_without_migrating_source(tmp_pa assert capture.task_description == "sealed capture" assert capture.capture_dir != capture_dir assert capture.capture_dir.parent != capture_dir.parent - assert json.loads( - (capture.capture_dir / CAPTURE_TERMINAL_FILENAME).read_text() - )["terminal_sha256"] == terminal.terminal_sha256 + assert ( + json.loads((capture.capture_dir / CAPTURE_TERMINAL_FILENAME).read_text())[ + "terminal_sha256" + ] + == terminal.terminal_sha256 + ) after = (source_db.stat().st_mtime_ns, hashlib.sha256(source_db.read_bytes()).hexdigest()) assert after == before @@ -241,6 +386,104 @@ def test_verified_loader_rejects_duplicate_source_ordinals(tmp_path) -> None: CaptureSession.load_verified(capture_dir) +def test_verified_loader_requires_a_v2_after_frame_for_every_action(tmp_path) -> None: + capture_dir = _v2_capture_directory(tmp_path, frame_ordinals=(1,)) + + with pytest.raises(ValueError, match="no ordinal-later retained after frame"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_accepts_a_complete_v2_source_journal(tmp_path) -> None: + capture_dir = _v2_capture_directory(tmp_path) + + with CaptureSession.load_verified(capture_dir) as capture: + assert [frame.source_ordinal for frame in capture.frames()] == [1, 3] + + +def test_verified_loader_rejects_a_gap_in_the_v2_source_journal(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 3), + action_ordinal=None, + ) + + with pytest.raises(ValueError, match="source journal has a missing ordinal"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_recomputes_retained_v2_png_digest(tmp_path) -> None: + capture_dir = _v2_capture_directory(tmp_path, wrong_png_digest=True) + + with pytest.raises(ValueError, match="PNG digest differs"): + CaptureSession.load_verified(capture_dir) + + +@pytest.mark.parametrize( + "timing", + [ + ( + None, + [(0, 0.0), (1, 1.0)], + [(0, 11.0), (1, 12.0)], + [(0, 1), (1, 4)], + ), + ( + None, + [(0, 0.0), (1, 1.0)], + [(0, 11.0), (1, 99.0)], + [(0, 1), (1, 3)], + ), + ], +) +def test_verified_loader_joins_v2_mp4_bindings_to_database_frames( + tmp_path, + monkeypatch, + timing, +) -> None: + capture_dir = _v2_capture_directory(tmp_path, video=True) + monkeypatch.setattr("openadapt_capture.video._read_timing_metadata", lambda _path: timing) + + with pytest.raises(ValueError, match="MP4 source bindings|capture-time bindings"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_accepts_v2_mp4_bindings_joined_to_database_frames( + tmp_path, + monkeypatch, +) -> None: + capture_dir = _v2_capture_directory(tmp_path, video=True) + timing = ( + None, + [(0, 0.0), (1, 1.0)], + [(0, 11.0), (1, 12.0)], + [(0, 1), (1, 3)], + ) + monkeypatch.setattr("openadapt_capture.video._read_timing_metadata", lambda _path: timing) + + with CaptureSession.load_verified(capture_dir) as capture: + assert capture.video_path is not None + + +def test_verified_loader_rejects_multiple_recognized_video_artifacts(tmp_path) -> None: + capture_dir = _v2_capture_directory(tmp_path, video=True) + # Add the second recognized name before replacing the immutable seal. + (capture_dir / CAPTURE_TERMINAL_FILENAME).unlink() + (capture_dir / ARTIFACT_MANIFEST_FILENAME).unlink() + (capture_dir / "oa_recording-10.mp4").write_bytes(b"fixture") + seal_capture( + capture_dir, + session_id="v2-session", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={"action": 1, "screen": 2, "window": 2, "browser": 0, "video": 2}, + last_source_ordinal=3, + ) + + with pytest.raises(ValueError, match="multiple recognized MP4"): + CaptureSession.load_verified(capture_dir) + + def test_verified_loader_opens_an_encoded_immutable_database_uri(tmp_path) -> None: capture_dir = _capture_directory(tmp_path / "capture#fragment") _seal(capture_dir) diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 3a43bb8..71a9203 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -156,7 +156,7 @@ def reserve_action(): assert frame.data.geometry_generation == 2 -def test_input_observation_waits_for_an_in_flight_window_frame(fake): +def test_input_observation_precedes_an_in_flight_window_frame(fake): capture_entered = threading.Event() release_capture = threading.Event() terminate = threading.Event() @@ -213,6 +213,7 @@ def capturer(window): assert capture_entered.wait(timeout=5) action_finished = threading.Event() + action_binding: dict[str, int] = {} def reserve_action(): action_timestamp = time.time() @@ -222,6 +223,7 @@ def reserve_action(): 310.0, 170.0, ) + action_binding["generation"] = binding[2] reservation.complete( Event( action_timestamp, @@ -233,7 +235,15 @@ def reserve_action(): action_reader = threading.Thread(target=reserve_action) action_reader.start() - assert not action_finished.wait(timeout=0.1) + assert action_finished.wait(timeout=5) + + # The in-flight frame can include the action's result. It must remain + # unpublished until after the action has bound the previous exact frame. + initial = journal.get_nowait() + action = journal.get_nowait() + assert [initial.type, action.type] == ["screen", "action"] + assert action_binding["generation"] == initial.data.geometry_generation + assert journal.empty() release_capture.set() screen_reader.join(timeout=5) @@ -241,11 +251,7 @@ def reserve_action(): assert not screen_reader.is_alive() assert not action_reader.is_alive() - assert [journal.get_nowait().type for _ in range(3)] == [ - "screen", - "screen", - "action", - ] + assert journal.get_nowait().type == "screen" def test_window_capture_state_rejects_scales_not_derived_from_content(scope): From 4bf541c8ae98f1cd3c60bd5f422bda35c2d8b555 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 26 Aug 2026 17:36:35 -0400 Subject: [PATCH 6/6] fix: bind native actions to exact frame intervals --- docs/DESIGN.md | 4 + openadapt_capture/capture.py | 283 ++++++++-- openadapt_capture/control.py | 16 +- openadapt_capture/db/models.py | 6 + openadapt_capture/events.py | 23 + openadapt_capture/input_observer/base.py | 320 ++++++++++- openadapt_capture/input_observer/darwin.py | 85 ++- openadapt_capture/input_observer/linux.py | 232 ++++++-- openadapt_capture/input_observer/windows.py | 114 +++- openadapt_capture/processing.py | 110 ++-- openadapt_capture/recorder.py | 577 +++++++++++++++++--- openadapt_capture/terminal.py | 43 +- openadapt_capture/window_capture.py | 56 ++ tests/test_capture_terminal.py | 294 +++++++++- tests/test_control.py | 101 ++++ tests/test_desktop_capture.py | 120 +++- tests/test_frame_binding.py | 62 ++- tests/test_highlevel.py | 14 + tests/test_input_observer.py | 219 ++++++++ tests/test_input_observer_darwin.py | 285 +++++++++- tests/test_input_observer_linux_xkb.py | 126 +++++ tests/test_input_observer_windows.py | 107 ++++ tests/test_processing.py | 67 +++ tests/test_window_capture.py | 481 ++++++++++++++++ 24 files changed, 3501 insertions(+), 244 deletions(-) diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 9ee7809..90e3d02 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -139,6 +139,10 @@ Platform observers have one ordered callback contract. They identify injected events when the operating system provides that information. Capture can exclude its own injected qualification events from a normal session. It refuses an incomplete observer startup instead of reporting partial coverage as complete. +On macOS, an active pass-through session tap holds downstream event delivery +only while Capture commits a clean frame. This requires both Input Monitoring +and Accessibility permission. The interactive macOS qualification proves that +an annotated downstream event cannot pass that cut before the frame commit. The post-processing layer merges primitive events into higher-level actions. The compiler remains responsible for refusing action forms that its selected diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index aac934f..3a71d5e 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -128,9 +128,24 @@ def _convert_action_event(db_event) -> PydanticActionEvent: ), "screenshot_timestamp": getattr(db_event, "screenshot_timestamp", None), "screenshot_source_ordinal": getattr(db_event, "screenshot_source_ordinal", None), + "after_screenshot_timestamp": getattr( + db_event, "after_screenshot_timestamp", None + ), + "after_screenshot_source_ordinal": getattr( + db_event, "after_screenshot_source_ordinal", None + ), "window_event_timestamp": getattr(db_event, "window_event_timestamp", None), "window_event_source_ordinal": getattr(db_event, "window_event_source_ordinal", None), + "after_window_event_timestamp": getattr( + db_event, "after_window_event_timestamp", None + ), + "after_window_event_source_ordinal": getattr( + db_event, "after_window_event_source_ordinal", None + ), "window_geometry_generation": getattr(db_event, "window_geometry_generation", None), + "after_window_geometry_generation": getattr( + db_event, "after_window_geometry_generation", None + ), } if db_event.name == "move": @@ -482,6 +497,16 @@ def screenshot_source_ordinal(self) -> int | None: """Return the exact source-journal position of the bound frame.""" return self.event.screenshot_source_ordinal + @property + def after_screenshot_timestamp(self) -> float | None: + """Exact retained after-frame timestamp bound to this action.""" + return self.event.after_screenshot_timestamp + + @property + def after_screenshot_source_ordinal(self) -> int | None: + """Return the source-journal position of the retained after frame.""" + return self.event.after_screenshot_source_ordinal + @property def window_event_timestamp(self) -> float | None: """Exact atomic WindowEvent timestamp bound to this action.""" @@ -492,11 +517,26 @@ def window_event_source_ordinal(self) -> int | None: """Return the source-journal position of the bound native geometry.""" return self.event.window_event_source_ordinal + @property + def after_window_event_timestamp(self) -> float | None: + """Exact native geometry timestamp paired with the after frame.""" + return self.event.after_window_event_timestamp + + @property + def after_window_event_source_ordinal(self) -> int | None: + """Return the geometry source position paired with the after frame.""" + return self.event.after_window_event_source_ordinal + @property def window_geometry_generation(self) -> int | None: """Exact native geometry generation bound to this action.""" return self.event.window_geometry_generation + @property + def after_window_geometry_generation(self) -> int | None: + """Exact native geometry generation paired with the after frame.""" + return self.event.after_window_geometry_generation + @property def screenshot(self) -> "Image" | None: """Get the exact retained screen frame this action is bound to. @@ -520,6 +560,21 @@ def screenshot(self) -> "Image" | None: ) return self._capture.get_frame_at(self.timestamp) + @property + def after_screenshot(self) -> "Image" | None: + """Get the exact first retained frame after this action.""" + bound = getattr(self.event, "after_screenshot_timestamp", None) + if bound is None: + return None + return self._capture.get_exact_frame( + bound, + source_ordinal=getattr( + self.event, + "after_screenshot_source_ordinal", + None, + ), + ) + def _source_order(rows: list) -> list: """Order current journal rows by ordinal and legacy rows by timestamp.""" @@ -574,6 +629,12 @@ def _validate_database_contract( if calculated_last != last_source_ordinal: raise InvalidCaptureEvent("sealed last source ordinal differs from the immutable database") + screenshots_by_ordinal = { + row.source_ordinal: row for row in rows_by_kind["screen"] + } + windows_rows_by_ordinal = { + row.source_ordinal: row for row in rows_by_kind["window"] + } for event in rows_by_kind["action"]: _convert_action_event(event) if event.screenshot_id is None or event.screenshot is None: @@ -592,11 +653,108 @@ def _validate_database_contract( or event.window_event_timestamp != event.window_event.timestamp ): raise InvalidCaptureEvent("sealed action window relationship is inconsistent") + if event.after_screenshot_source_ordinal is not None: + after_screenshot = screenshots_by_ordinal.get( + event.after_screenshot_source_ordinal + ) + if ( + after_screenshot is None + or event.after_screenshot_timestamp != after_screenshot.timestamp + or event.source_ordinal >= event.after_screenshot_source_ordinal + ): + raise InvalidCaptureEvent( + "sealed action after-screenshot binding is inconsistent" + ) + if event.after_window_event_source_ordinal is not None: + after_window = windows_rows_by_ordinal.get( + event.after_window_event_source_ordinal + ) + if ( + after_window is None + or event.after_window_event_timestamp != after_window.timestamp + ): + raise InvalidCaptureEvent( + "sealed action after-window binding is inconsistent" + ) + retained_frame_sizes: dict[int, tuple[int, int]] = {} + for screenshot in rows_by_kind["screen"]: + if screenshot.png_data: + from PIL import Image + + retained_sha256 = hashlib.sha256(screenshot.png_data).hexdigest() + if screenshot.png_sha256 != retained_sha256: + raise InvalidCaptureEvent( + "sealed frame PNG digest differs from its retained bytes" + ) + try: + with Image.open(io.BytesIO(screenshot.png_data)) as retained: + retained.load() + retained_frame_sizes[int(screenshot.source_ordinal)] = retained.size + except Exception as exc: + raise InvalidCaptureEvent("sealed frame PNG is invalid") from exc + elif screenshot.png_sha256 is not None: + raise InvalidCaptureEvent( + "sealed frame has a PNG digest without retained PNG bytes" + ) + metadata = capture.window_capture is_v2 = ( isinstance(metadata, dict) and metadata.get("schema_version") == "openadapt.capture.window-scoped/v2" ) + desktop_metadata = capture.desktop_capture + is_desktop_v1 = ( + isinstance(desktop_metadata, dict) + and desktop_metadata.get("schema_version") + == "openadapt.capture.display-topology/v1" + and desktop_metadata.get("coordinate_space") == "virtual_desktop_pixels" + ) + is_native_scoped = is_v2 or is_desktop_v1 + if is_native_scoped and not rows_by_kind["screen"]: + raise InvalidCaptureEvent("sealed native capture has no retained frames") + if is_desktop_v1: + viewport = desktop_metadata.get("viewport") + if ( + not isinstance(viewport, list) + or len(viewport) != 2 + or any(isinstance(value, bool) or not isinstance(value, int) for value in viewport) + or any(value <= 0 for value in viewport) + ): + raise InvalidCaptureEvent("sealed desktop capture has an invalid viewport") + expected_size = tuple(viewport) + if any(size != expected_size for size in retained_frame_sizes.values()): + raise InvalidCaptureEvent( + "sealed desktop frame dimensions differ from its capture viewport" + ) + + if is_native_scoped: + frame_ordinals = set(screenshots_by_ordinal) + for action in rows_by_kind["action"]: + if action.after_screenshot_source_ordinal is None: + raise InvalidCaptureEvent( + "sealed native action has an incomplete before/after frame binding" + ) + before_candidates = [ + ordinal for ordinal in frame_ordinals if ordinal < action.source_ordinal + ] + after_candidates = [ + ordinal for ordinal in frame_ordinals if ordinal > action.source_ordinal + ] + if ( + not before_candidates + or action.screenshot_source_ordinal != max(before_candidates) + ): + raise InvalidCaptureEvent( + "sealed native action does not bind its nearest retained before frame" + ) + if ( + not after_candidates + or action.after_screenshot_source_ordinal != min(after_candidates) + ): + raise InvalidCaptureEvent( + "sealed native action does not bind its first retained after frame" + ) + allowed_pair_ordinals: set[int] = set() if is_v2: window_events = capture.window_capture_events_v2() @@ -616,27 +774,10 @@ def _validate_database_contract( screenshot = screenshots_by_ordinal[ordinal] if screenshot.timestamp != window_event.timestamp: raise InvalidCaptureEvent("sealed v2 frame and geometry timestamps differ") - if screenshot.png_data: - from PIL import Image - - retained_sha256 = hashlib.sha256(screenshot.png_data).hexdigest() - if screenshot.png_sha256 != retained_sha256: - raise InvalidCaptureEvent( - "sealed v2 frame PNG digest differs from its retained bytes" - ) - try: - with Image.open(io.BytesIO(screenshot.png_data)) as retained: - retained.load() - retained_size = retained.size - except Exception as exc: - raise InvalidCaptureEvent("sealed v2 frame PNG is invalid") from exc - if retained_size != state.viewport: - raise InvalidCaptureEvent( - "sealed v2 frame dimensions differ from its geometry viewport" - ) - elif screenshot.png_sha256 is not None: + retained_size = retained_frame_sizes.get(int(screenshot.source_ordinal)) + if retained_size is not None and retained_size != state.viewport: raise InvalidCaptureEvent( - "sealed v2 frame has a PNG digest without retained PNG bytes" + "sealed v2 frame dimensions differ from its geometry viewport" ) current_identity = ( state.window_id, @@ -664,20 +805,32 @@ def _validate_database_contract( if ( action.window_geometry_generation is None or action.screenshot_source_ordinal != action.window_event_source_ordinal + or action.after_screenshot_source_ordinal is None + or action.after_window_event_source_ordinal is None + or action.after_screenshot_source_ordinal + != action.after_window_event_source_ordinal + or action.after_screenshot_timestamp + != action.after_window_event_timestamp + or action.after_window_geometry_generation is None ): - raise InvalidCaptureEvent("sealed v2 action has an incomplete frame binding") + raise InvalidCaptureEvent( + "sealed v2 action has an incomplete before/after frame binding" + ) bound = windows_by_ordinal.get(action.window_event_source_ordinal) if bound is None or ( bound.window_capture_v2.geometry_generation != action.window_geometry_generation ): raise InvalidCaptureEvent("sealed v2 action names the wrong geometry epoch") - if not any( - frame_ordinal > action.source_ordinal for frame_ordinal in allowed_pair_ordinals + after_bound = windows_by_ordinal.get( + action.after_window_event_source_ordinal + ) + if after_bound is None or ( + after_bound.window_capture_v2.geometry_generation + != action.after_window_geometry_generation ): raise InvalidCaptureEvent( - "sealed v2 action has no ordinal-later retained after frame" + "sealed v2 action names the wrong after-frame geometry epoch" ) - owners: dict[int, set[str]] = {} for kind, ordinals in ordinals_by_kind.items(): for ordinal in ordinals: @@ -687,12 +840,22 @@ def _validate_database_contract( ordinal in allowed_pair_ordinals and kinds == {"screen", "window"} ): raise InvalidCaptureEvent("sealed journal reuses a source ordinal across events") - if is_v2 and sorted(owners) != list(range(1, (last_source_ordinal or 0) + 1)): - raise InvalidCaptureEvent("sealed v2 source journal has a missing ordinal") + if is_native_scoped and sorted(owners) != list( + range(1, (last_source_ordinal or 0) + 1) + ): + raise InvalidCaptureEvent("sealed native source journal has a missing ordinal") expected_video_count = event_counts.get("video") if not isinstance(expected_video_count, int) or expected_video_count < 0: raise InvalidCaptureEvent("sealed video count is invalid") + if ( + is_native_scoped + and not expected_video_count + and any(not screenshot.png_data for screenshot in rows_by_kind["screen"]) + ): + raise InvalidCaptureEvent( + "sealed native frame has no retained PNG or exact MP4 carrier" + ) if expected_video_count: from openadapt_capture.video import _read_timing_metadata @@ -704,27 +867,30 @@ def _validate_database_contract( raise InvalidCaptureEvent("sealed MP4 has no source-ordinal frame bindings") if len(timing[3]) != expected_video_count: raise InvalidCaptureEvent("sealed video count differs from its MP4 bindings") - if is_v2: + if is_native_scoped: video_source_ordinals = [source_ordinal for _, source_ordinal in timing[3]] expected_source_ordinals = sorted(ordinals_by_kind["screen"]) if video_source_ordinals != expected_source_ordinals: raise InvalidCaptureEvent( - "sealed v2 MP4 source bindings differ from retained frame ordinals" + "sealed native MP4 source bindings differ from retained frame ordinals" ) capture_bindings = timing[2] if capture_bindings is None: - raise InvalidCaptureEvent("sealed v2 MP4 has no exact capture-time frame bindings") - capture_by_index = dict(capture_bindings) + raise InvalidCaptureEvent( + "sealed native MP4 has no exact capture-time frame bindings" + ) screenshots_by_ordinal = { int(row.source_ordinal): row for row in rows_by_kind["screen"] } - for encoded_index, source_ordinal in timing[3]: - screenshot = screenshots_by_ordinal[source_ordinal] - if capture_by_index.get(encoded_index) != screenshot.timestamp: - raise InvalidCaptureEvent( - "sealed v2 MP4 source and capture-time bindings differ " - "from the retained frame" - ) + expected_capture_bindings = [ + (encoded_index, screenshots_by_ordinal[source_ordinal].timestamp) + for encoded_index, source_ordinal in timing[3] + ] + if capture_bindings != expected_capture_bindings: + raise InvalidCaptureEvent( + "sealed native MP4 source and capture-time bindings differ " + "from the retained frame" + ) elif capture.video_path is not None: raise InvalidCaptureEvent("sealed capture inventories an MP4 but claims no video frames") @@ -848,6 +1014,47 @@ def _discard() -> None: raise return result + @classmethod + def validate_sealed(cls, capture_dir: str | Path) -> None: + """Validate one seal with a database-only temporary snapshot.""" + from openadapt_capture.db import get_immutable_session_for_path + from openadapt_capture.db.models import Recording + from openadapt_capture.terminal import ( + CaptureSealError, + copy_verified_database, + verify_capture_artifacts, + ) + + source = Path(capture_dir).resolve() + temporary, database_path, terminal, manifest = copy_verified_database(source) + session = None + try: + session = get_immutable_session_for_path(str(database_path)) + recordings = session.query(Recording).all() + if len(recordings) != 1: + raise InvalidCaptureEvent( + "verified capture must contain exactly one recording, " + f"found {len(recordings)}" + ) + capture = cls(source, session, recordings[0]) + capture._verified_terminal = terminal + _validate_database_contract( + capture, + event_counts=terminal.event_counts.model_dump(mode="python"), + last_source_ordinal=terminal.last_source_ordinal, + ) + finally: + if session is not None: + bind = session.get_bind() + session.close() + if bind is not None: + bind.dispose() + temporary.cleanup() + + final_terminal, final_manifest = verify_capture_artifacts(source) + if final_terminal != terminal or final_manifest != manifest: + raise CaptureSealError("capture seal changed during semantic validation") + @property def terminal(self): """Return the verified immutable terminal, or None for a legacy load.""" diff --git a/openadapt_capture/control.py b/openadapt_capture/control.py index 0cf84f3..68547be 100644 --- a/openadapt_capture/control.py +++ b/openadapt_capture/control.py @@ -782,9 +782,23 @@ def _mark_crashed_if_bound(descriptor: _ControlDescriptor) -> None: or payload.get("session_id") != descriptor.session_id or payload.get("pid") != descriptor.pid or payload.get("process_started_at") != descriptor.process_started_at - or payload.get("complete") is True ): return + if payload.get("complete") is True: + from openadapt_capture.terminal import ( + CaptureSealError, + verify_capture_artifacts, + ) + + try: + verify_capture_artifacts(descriptor.capture_dir) + except (CaptureSealError, OSError, ValueError): + # The process exited between staging the final state and sealing. + # A success-shaped state file without its exact immutable seal is + # not a completed capture. + pass + else: + return payload.update( { "phase": "crashed", diff --git a/openadapt_capture/db/models.py b/openadapt_capture/db/models.py index 26101b5..eedba8e 100644 --- a/openadapt_capture/db/models.py +++ b/openadapt_capture/db/models.py @@ -103,9 +103,13 @@ class ActionEvent(Base): screenshot_timestamp = sa.Column(ForceFloat) screenshot_source_ordinal = sa.Column(sa.Integer) screenshot_id = sa.Column(sa.ForeignKey("screenshot.id")) + after_screenshot_timestamp = sa.Column(ForceFloat) + after_screenshot_source_ordinal = sa.Column(sa.Integer) window_event_timestamp = sa.Column(ForceFloat) window_event_source_ordinal = sa.Column(sa.Integer) window_event_id = sa.Column(sa.ForeignKey("window_event.id")) + after_window_event_timestamp = sa.Column(ForceFloat) + after_window_event_source_ordinal = sa.Column(sa.Integer) browser_event_timestamp = sa.Column(ForceFloat) browser_event_id = sa.Column(sa.ForeignKey("browser_event.id")) mouse_x = sa.Column(sa.Numeric(asdecimal=False)) @@ -133,6 +137,8 @@ class ActionEvent(Base): # Exact native geometry generation bound to the action's retained frame. # Nullable keeps legacy and full-desktop captures readable. window_geometry_generation = sa.Column(sa.Integer) + # Exact geometry generation paired with the first retained after frame. + after_window_geometry_generation = sa.Column(sa.Integer) disabled = sa.Column(sa.Boolean, default=False) children = sa.orm.relationship("ActionEvent") diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 907892d..996484d 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -205,6 +205,15 @@ class ActionBaseEvent(BaseEvent): ge=1, description="Source journal ordinal of the exact retained screen frame", ) + after_screenshot_timestamp: float | None = Field( + default=None, + description="Exact timestamp of the first retained frame after this action", + ) + after_screenshot_source_ordinal: int | None = Field( + default=None, + ge=1, + description="Source journal ordinal of the exact retained after frame", + ) window_event_timestamp: float | None = Field( default=None, description="Exact WindowEvent timestamp paired with the bound frame", @@ -214,11 +223,25 @@ class ActionBaseEvent(BaseEvent): ge=1, description="Source journal ordinal of the geometry paired with the frame", ) + after_window_event_timestamp: float | None = Field( + default=None, + description="Exact WindowEvent timestamp paired with the retained after frame", + ) + after_window_event_source_ordinal: int | None = Field( + default=None, + ge=1, + description="Source journal ordinal of geometry paired with the after frame", + ) window_geometry_generation: int | None = Field( default=None, ge=1, description="Exact native geometry generation bound to this action", ) + after_window_geometry_generation: int | None = Field( + default=None, + ge=1, + description="Exact native geometry generation paired with the after frame", + ) # ============================================================================= diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index d3da66b..ec2fd30 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -80,6 +80,29 @@ class ObservedKey: InputCallback: TypeAlias = Callable[[ObservedInput], None] +@dataclass(frozen=True, slots=True) +class _ObservedDelivery: + """One normalized event plus its native-receipt reservation.""" + + event: ObservedInput + receipt: object | None + + +@dataclass(slots=True) +class _ReceiptFrameCapture: + """One platform-neutral cut around window-pixel acquisition.""" + + start_receipt_count: int + start_callback_generation: int + started_with_active_callback: bool + finish_called: bool = False + cut_held: bool = False + completed: bool = False + + +_SEALED_FRAME_RECEIPT = object() + + def add_exception_note(error: BaseException, note: str) -> None: """Attach cleanup context when supported without masking the primary error.""" add_note = getattr(error, "add_note", None) @@ -102,6 +125,23 @@ def check_health(self) -> None: def stop(self) -> None: """Stop and join the observer, surfacing any observer failure.""" + def begin_frame_capture(self) -> object | None: + """Start an input-stable frame capture transaction.""" + return None + + def finish_frame_capture(self, token: object | None) -> bool: + """Close the capture interval and report whether it contained input.""" + del token + return True + + def complete_frame_capture(self, token: object | None) -> None: + """Release an input-stable frame after it is published or discarded.""" + del token + + def seal_frame_capture(self, token: object | None) -> None: + """Stop accepting native input at an exact terminal-frame boundary.""" + del token + def __enter__(self) -> "InputObserver": self.start() return self @@ -139,10 +179,11 @@ def __init__( self._stop_requested = threading.Event() self._thread: threading.Thread | None = None self._delivery_thread: threading.Thread | None = None - self._delivery_queue: queue.Queue[ObservedInput | object] = queue.Queue( + self._delivery_queue: queue.Queue[_ObservedDelivery | object] = queue.Queue( maxsize=delivery_queue_size ) self._delivery_sentinel = object() + self._delivery_ready = threading.Event() self._delivery_stop_requested = threading.Event() self._delivery_decided = threading.Event() self._delivery_state_lock = threading.Lock() @@ -150,6 +191,12 @@ def __init__( self._failure: BaseException | None = None self._failure_lock = threading.Lock() self._startup_failure: BaseException | None = None + self._receipt_lock = threading.Lock() + self._unqueued_receipts: list[object] = [] + self._input_receipt_count = 0 + self._native_callback_generation = 0 + self._active_native_callbacks = 0 + self._capture_sealed = False @abstractmethod def _setup(self) -> None: @@ -166,29 +213,205 @@ def _teardown(self) -> None: def _wake(self) -> None: """Wake a blocked event loop during shutdown, when needed.""" - def _emit(self, event: ObservedInput) -> None: + def _reserve_receipt(self, timestamp: float | None) -> object | None: + """Reserve source order at the first native receipt boundary.""" + reserve = getattr(self.callback, "_openadapt_input_receipt", None) + with self._receipt_lock: + if self._capture_sealed: + return _SEALED_FRAME_RECEIPT + self._input_receipt_count += 1 + if not callable(reserve): + return None + if timestamp is None: + raise InputObserverError( + "native input has no receipt timestamp for source-order reservation" + ) + receipt = reserve(timestamp) + if receipt is not None: + self._unqueued_receipts.append(receipt) + return receipt + + def _mark_native_activity(self) -> None: + """Make an unrecorded native transition invalidate an in-flight frame.""" + with self._receipt_lock: + if not self._capture_sealed: + self._input_receipt_count += 1 + + def _begin_native_callback(self) -> None: + """Mark an OS input callback active before it can deliver its input.""" + with self._receipt_lock: + self._native_callback_generation += 1 + self._active_native_callbacks += 1 + + def _end_native_callback(self) -> None: + """Close an OS input callback only after its downstream hook returns.""" + with self._receipt_lock: + if self._active_native_callbacks <= 0: + raise InputObserverError( + "native input callback completion has no matching start" + ) + self._active_native_callbacks -= 1 + self._native_callback_generation += 1 + + @staticmethod + def _fail_receipt(receipt: object | None, failure: BaseException) -> None: + """Release a reserved consumer position after observation fails.""" + if receipt is None: + return + fail = getattr(receipt, "fail", None) + if callable(fail): + fail(failure) + + def _forget_unqueued_receipt(self, receipt: object | None) -> None: + if receipt is None: + return + with self._receipt_lock: + for index, pending in enumerate(self._unqueued_receipts): + if pending is receipt: + self._unqueued_receipts.pop(index) + return + + def _fail_unqueued_receipts(self, failure: BaseException) -> int: + """Fail reservations that normalization has not queued for delivery.""" + with self._receipt_lock: + pending = self._unqueued_receipts + self._unqueued_receipts = [] + for receipt in pending: + self._fail_receipt(receipt, failure) + return len(pending) + + def _emit( + self, + event: ObservedInput, + *, + receipt: object | None = None, + ) -> None: + if receipt is _SEALED_FRAME_RECEIPT: + return + if receipt is None: + with self._receipt_lock: + if self._capture_sealed: + return + with self._delivery_state_lock: + if self._delivery_state in {"aborted", "inactive"}: + if receipt is None: + return + failure = InputObserverError( + f"{type(self).__name__} discarded reserved input after " + "delivery stopped" + ) + self._forget_unqueued_receipt(receipt) + self._fail_receipt(receipt, failure) + raise failure + # Keep the state check and enqueue atomic with startup cancellation. # Otherwise an abort could drain the queue and exit its delivery thread # between these two operations, leaving a late setup event stranded. failure: InputObserverError | None = None with self._delivery_state_lock: if self._delivery_state in {"aborted", "inactive"}: - return - try: - self._delivery_queue.put_nowait(event) - except queue.Full: failure = InputObserverError( - f"{type(self).__name__} input delivery queue overflowed; " - "recording coverage is incomplete" + f"{type(self).__name__} discarded input after delivery stopped" ) + else: + self._forget_unqueued_receipt(receipt) + try: + self._delivery_queue.put_nowait( + _ObservedDelivery(event, receipt) + ) + except queue.Full: + failure = InputObserverError( + f"{type(self).__name__} input delivery queue overflowed; " + "recording coverage is incomplete" + ) if failure is not None: + self._fail_receipt(receipt, failure) # Wake hooks are platform-defined and may re-enter lifecycle code; # never call them while holding the delivery-state lock. self._fail(failure) raise failure + def _emit_received( + self, + event: ObservedInput, + receipt: object | None, + ) -> None: + """Queue one normalized event with its optional receipt reservation.""" + if receipt is _SEALED_FRAME_RECEIPT: + return + if receipt is None: + self._emit(event) + else: + self._emit(event, receipt=receipt) + + def begin_frame_capture(self) -> object: + """Snapshot accepted input before the screen reader acquires pixels.""" + self.check_health() + with self._receipt_lock: + if self._capture_sealed: + raise InputObserverError( + "native input was already sealed before frame capture" + ) + return _ReceiptFrameCapture( + start_receipt_count=self._input_receipt_count, + start_callback_generation=self._native_callback_generation, + started_with_active_callback=self._active_native_callbacks > 0, + ) + + def finish_frame_capture(self, token: object | None) -> bool: + """Reject an input-dirty frame and hold a clean cut through commit.""" + if not isinstance(token, _ReceiptFrameCapture): + raise InputObserverError("native input received an invalid frame token") + if token.finish_called or token.completed: + raise InputObserverError("native input frame token was already finished") + if not self._receipt_lock.acquire(timeout=self.shutdown_timeout): + raise InputObserverError( + "native input could not close the frame cut before timeout" + ) + token.finish_called = True + token.cut_held = True + try: + self.check_health() + if ( + token.started_with_active_callback + or self._active_native_callbacks > 0 + or self._native_callback_generation + != token.start_callback_generation + or self._input_receipt_count != token.start_receipt_count + ): + token.cut_held = False + self._receipt_lock.release() + return False + return True + except BaseException: + token.cut_held = False + self._receipt_lock.release() + raise + + def complete_frame_capture(self, token: object | None) -> None: + """Release native callbacks after a clean frame enters the journal.""" + if not isinstance(token, _ReceiptFrameCapture) or token.completed: + return + token.completed = True + if token.cut_held: + token.cut_held = False + self._receipt_lock.release() + + def seal_frame_capture(self, token: object | None) -> None: + """Close native receipt acceptance before a terminal frame commits.""" + if isinstance(token, _ReceiptFrameCapture): + if not token.cut_held or token.completed: + raise InputObserverError( + "terminal input seal requires an active clean frame cut" + ) + self._capture_sealed = True + return + with self._receipt_lock: + self._capture_sealed = True + def _fail(self, failure: BaseException) -> None: primary = self._store_failure(failure) + self._fail_unqueued_receipts(primary) self._stop_requested.set() try: self._wake() @@ -217,7 +440,7 @@ def _delivery_main(self) -> None: self._discard_delivery_queue() return if delivery_state != "committed": - self._fail( + self._abort_failed_delivery( InputObserverError( f"{type(self).__name__} delivery started without a valid " "startup decision" @@ -237,6 +460,7 @@ def _delivery_main(self) -> None: try: if callable(start_hook): start_hook() + self._delivery_ready.set() while True: if ( self._delivery_stop_requested.is_set() @@ -250,7 +474,30 @@ def _delivery_main(self) -> None: try: if item is self._delivery_sentinel: return - self.callback(item) # type: ignore[arg-type] + if not isinstance(item, _ObservedDelivery): + raise InputObserverError( + f"{type(self).__name__} delivery queue contained " + "an invalid item" + ) + if item.receipt is None: + self.callback(item.event) + else: + deliver = getattr( + self.callback, + "_openadapt_input_delivery", + None, + ) + if not callable(deliver): + raise InputObserverError( + "a native input receipt was reserved without " + "a matching delivery consumer" + ) + deliver(item.event, item.receipt) + if not bool(getattr(item.receipt, "finished", False)): + raise InputObserverError( + "the input consumer returned without completing " + "its native receipt reservation" + ) except BaseException as exc: failure = ( exc @@ -259,7 +506,9 @@ def _delivery_main(self) -> None: f"{type(self).__name__} input consumer failed: {exc}" ) ) - self._fail(failure) + if isinstance(item, _ObservedDelivery): + self._fail_receipt(item.receipt, failure) + self._abort_failed_delivery(failure) return finally: self._delivery_queue.task_done() @@ -271,7 +520,7 @@ def _delivery_main(self) -> None: f"{type(self).__name__} input delivery setup failed: {exc}" ) ) - self._fail(failure) + self._abort_failed_delivery(failure) finally: if callable(stop_hook): try: @@ -283,14 +532,33 @@ def _delivery_main(self) -> None: ) ) - def _discard_delivery_queue(self) -> None: + def _abort_failed_delivery(self, failure: BaseException) -> None: + """Close delivery and fail every receipt that cannot reach its consumer.""" + with self._delivery_state_lock: + self._delivery_state = "aborted" + self._delivery_decided.set() + self._delivery_stop_requested.set() + self._fail(failure) + self._discard_delivery_queue(failure) + self._delivery_ready.set() + + def _discard_delivery_queue( + self, + failure: BaseException | None = None, + ) -> None: """Discard every event buffered by a startup that did not commit.""" + if failure is None: + failure = InputObserverError( + f"{type(self).__name__} discarded input from an aborted startup" + ) while True: try: - self._delivery_queue.get_nowait() + item = self._delivery_queue.get_nowait() except queue.Empty: return else: + if isinstance(item, _ObservedDelivery): + self._fail_receipt(item.receipt, failure) self._delivery_queue.task_done() def _abort_delivery_start(self) -> None: @@ -324,6 +592,7 @@ def _start_delivery(self) -> None: "previous lifecycle; stop it or create a new observer" ) self._delivery_queue = queue.Queue(maxsize=self.delivery_queue_size) + self._delivery_ready.clear() self._delivery_stop_requested.clear() self._delivery_decided.clear() with self._delivery_state_lock: @@ -461,6 +730,17 @@ def start(self) -> None: self._commit_delivery_start() except BaseException as exc: self._abort_start(exc) + if not self._delivery_ready.wait(self.startup_timeout): + self._abort_start( + InputObserverError( + f"{type(self).__name__} delivery setup did not become ready within " + f"{self.startup_timeout:.1f}s" + ) + ) + try: + self.check_health() + except BaseException as exc: + self._abort_start(exc) def _abort_start( self, @@ -517,6 +797,12 @@ def _abort_start( ) else: self._thread = None + incomplete_receipt = InputObserverError( + f"{type(self).__name__} stopped before reserved native input " + "reached delivery" + ) + if self._fail_unqueued_receipts(incomplete_receipt): + self._store_failure(incomplete_receipt) self._stop_delivery() raise primary @@ -563,6 +849,12 @@ def stop(self) -> None: ) else: self._thread = None + incomplete_receipt = InputObserverError( + f"{type(self).__name__} stopped before reserved native input " + "reached delivery" + ) + if self._fail_unqueued_receipts(incomplete_receipt): + self._store_failure(incomplete_receipt) self._stop_delivery() if primary is not None: with self._failure_lock: diff --git a/openadapt_capture/input_observer/darwin.py b/openadapt_capture/input_observer/darwin.py index 09e07f0..ee95548 100644 --- a/openadapt_capture/input_observer/darwin.py +++ b/openadapt_capture/input_observer/darwin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import threading import time from typing import Any @@ -116,6 +117,8 @@ def __init__( ) self._quartz = _quartz self._application_services = _application_services + self._delivery_barrier_lock = threading.Lock() + self._pending_delivery_barriers = 0 self._event_tap: Any | None = None self._cf_run_loop: Any | None = None self._run_loop_source: Any | None = None @@ -153,6 +156,11 @@ def _setup(self) -> None: "macOS denied global input observation. Enable Input Monitoring " "for OpenAdapt in System Settings > Privacy & Security." ) + if not self._has_active_filter_permission(): + raise InputObserverPermissionError( + "macOS denied the ordered input barrier. Enable Accessibility " + "for OpenAdapt in System Settings > Privacy & Security." + ) event_mask = 0 for event_type in self._observed_event_types(): @@ -160,9 +168,9 @@ def _setup(self) -> None: self._tap_callback_ref = self._event_callback self._event_tap = quartz.CGEventTapCreate( - quartz.kCGHIDEventTap, + quartz.kCGSessionEventTap, quartz.kCGHeadInsertEventTap, - quartz.kCGEventTapOptionListenOnly, + quartz.kCGEventTapOptionDefault, event_mask, self._tap_callback_ref, None, @@ -174,7 +182,7 @@ def _setup(self) -> None: "OpenAdapt was starting" ) raise InputObserverUnavailableError( - "macOS could not create a listen-only Quartz event tap" + "macOS could not create an active Quartz event barrier" ) self._cf_run_loop = quartz.CFRunLoopGetCurrent() @@ -217,6 +225,19 @@ def _has_observation_permission(self) -> bool: "macOS does not expose an input-observation permission API" ) + def _has_active_filter_permission(self) -> bool: + accessibility_check = getattr( + self._application_services, + "AXIsProcessTrusted", + None, + ) + if callable(accessibility_check): + return bool(accessibility_check()) + raise InputObserverUnavailableError( + "macOS does not expose the Accessibility permission required for " + "ordered input" + ) + def _observed_event_types(self) -> tuple[int, ...]: quartz = self._quartz event_types: list[int] = [] @@ -259,6 +280,21 @@ def _run_loop(self) -> None: 0.1, False, ) + self._complete_delivery_barriers() + self._complete_delivery_barriers() + + def _defer_delivery_barrier(self) -> None: + """Keep the callback active until control returns from the CF run loop.""" + with self._delivery_barrier_lock: + self._pending_delivery_barriers += 1 + + def _complete_delivery_barriers(self) -> None: + """Release callbacks only after Quartz has accepted their returned events.""" + with self._delivery_barrier_lock: + pending = self._pending_delivery_barriers + self._pending_delivery_barriers = 0 + for _ in range(pending): + self._end_native_callback() def _wake(self) -> None: if self._cf_run_loop is not None and self._quartz is not None: @@ -311,7 +347,11 @@ def _event_callback( ) return event + callback_entered = False try: + if event_type in self._observed_event_types(): + self._begin_native_callback() + callback_entered = True self._handle_event(event_type, event, timestamp=time.time()) except BaseException as exc: failure = ( @@ -320,6 +360,23 @@ def _event_callback( else InputObserverError(f"macOS input callback failed: {exc}") ) self._fail(failure) + finally: + if callback_entered: + try: + # This callback is an active filter at the head of the + # session event stream. Quartz cannot deliver the returned + # event while Python is still inside this function. Keep + # the frame barrier active until CFRunLoopRunInMode returns. + self._defer_delivery_barrier() + except BaseException as exc: + failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError( + f"macOS input callback completion failed: {exc}" + ) + ) + self._fail(failure) return event def _handle_event( @@ -335,6 +392,14 @@ def _handle_event( # for direct normalization calls makes the pure helper independently # testable without inventing a second observation moment. observed_at = timestamp + observed_type = event_type in self._observed_event_types() + if injected and observed_type: + self._mark_native_activity() + receipt = None + elif observed_type: + receipt = self._reserve_receipt(observed_at) + else: + receipt = None if event_type in self._mouse_move_event_types(): if self.observe_mouse and self.capture_mouse_moves: @@ -345,7 +410,8 @@ def _handle_event( y=y, injected=injected, timestamp=observed_at, - ) + ), + receipt=receipt, ) return @@ -356,7 +422,7 @@ def _handle_event( timestamp=observed_at, ) if button_event is not None: - self._emit(button_event) + self._emit(button_event, receipt=receipt) return if event_type == quartz.kCGEventScrollWheel and self.observe_mouse: @@ -379,7 +445,8 @@ def _handle_event( ), injected=injected, timestamp=observed_at, - ) + ), + receipt=receipt, ) return @@ -391,7 +458,8 @@ def _handle_event( pressed=event_type == quartz.kCGEventKeyDown, injected=injected, timestamp=observed_at, - ) + ), + receipt=receipt, ) return @@ -409,7 +477,8 @@ def _handle_event( injected=injected, include_character=False, timestamp=observed_at, - ) + ), + receipt=receipt, ) def _modifier_pressed(self, event: Any, keycode: int) -> bool: diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 53b952a..6db0bcd 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -19,6 +19,7 @@ import ctypes.util import os import sys +import threading import time from dataclasses import dataclass from typing import Any @@ -66,6 +67,15 @@ _XKB_COMPOSE_COMPOSED = 2 _XKB_COMPOSE_CANCELLED = 3 + +@dataclass(slots=True) +class _FrameCaptureCut: + """One pixel-capture interval closed by an ordered X RECORD marker.""" + + start_device_count: int + marker_seen: threading.Event + release_marker: threading.Event + _SPECIAL_KEY_NAMES = { "Alt_L": "alt", "Alt_R": "alt_r", @@ -220,6 +230,7 @@ class _PendingDeviceEvent: receipt_timestamp: float deadline: float candidate: _DeliveredCandidate | None = None + receipt: object | None = None def _event_byteorder(*, client_swapped: bool) -> str: @@ -375,6 +386,10 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._compose_context: Any = None self._compose_table: Any = None self._compose_state: Any = None + self._device_event_count = 0 + self._frame_cut_serial = threading.Lock() + self._frame_cut_state_lock = threading.Lock() + self._frame_cut: _FrameCaptureCut | None = None def _setup(self) -> None: if not sys.platform.startswith("linux"): @@ -736,20 +751,27 @@ def _record_intercept( f"{byte_length} bytes" ) payload = ctypes.string_at(recorded.data, byte_length) - if ( - self._waiting_baseline_marker - and int(recorded.id_base) == self._control_id_base - and payload[0] == 1 - ): - self._waiting_baseline_marker = False - if ( - self._stop_requested.is_set() - or self._startup_failure is not None - ): - self._accepting_events = False + if int(recorded.id_base) == self._control_id_base and payload[0] == 1: + if self._waiting_baseline_marker: + self._waiting_baseline_marker = False + if ( + self._stop_requested.is_set() + or self._startup_failure is not None + ): + self._accepting_events = False + return + self._baseline_marker_seen = True + self._accepting_events = True return - self._baseline_marker_seen = True - self._accepting_events = True + with self._frame_cut_state_lock: + frame_cut = self._frame_cut + if frame_cut is not None: + frame_cut.marker_seen.set() + if not frame_cut.release_marker.wait(timeout=self.shutdown_timeout): + raise InputObserverError( + "the screen reader did not complete the X RECORD " + "frame boundary before its deadline" + ) return if self._delivery_start_was_aborted(): # XRecordProcessReplies may have copied a complete native batch @@ -786,6 +808,7 @@ def _record_intercept( else: self._handle_delivered_event(event, id_base=int(recorded.id_base)) except BaseException as exc: + self._fail_unqueued_receipts(exc) if self._record_callback_failure is None: self._record_callback_failure = exc self._stop_requested.set() @@ -818,8 +841,94 @@ def _query_pointer_baseline(self) -> None: ) self._last_pointer = (float(root_x.value), float(root_y.value)) + def _send_frame_cut_marker(self) -> None: + """Place an ordered reply after all server input seen before this call.""" + root_return = ctypes.c_ulong() + child_return = ctypes.c_ulong() + root_x = ctypes.c_int() + root_y = ctypes.c_int() + window_x = ctypes.c_int() + window_y = ctypes.c_int() + mask = ctypes.c_uint() + if not self._x11.XQueryPointer( + self._control_display, + self._root, + ctypes.byref(root_return), + ctypes.byref(child_return), + ctypes.byref(root_x), + ctypes.byref(root_y), + ctypes.byref(window_x), + ctypes.byref(window_y), + ctypes.byref(mask), + ): + raise InputObserverError( + "X11 could not close the input interval around a captured frame" + ) + + def begin_frame_capture(self) -> object: + """Record the native-device count before the screen reader grabs pixels.""" + if not self._frame_cut_serial.acquire(timeout=self.shutdown_timeout): + raise InputObserverError("another X RECORD frame boundary did not complete") + try: + self.check_health() + if not self._setup_complete or not self._accepting_events: + raise InputObserverError( + "X RECORD cannot start a frame boundary before input is armed" + ) + with self._frame_cut_state_lock: + if self._frame_cut is not None: + raise InputObserverError("an X RECORD frame boundary is already active") + return _FrameCaptureCut( + start_device_count=self._device_event_count, + marker_seen=threading.Event(), + release_marker=threading.Event(), + ) + except BaseException: + self._frame_cut_serial.release() + raise + + def finish_frame_capture(self, token: object | None) -> bool: + """Flush through a marker and reject pixels concurrent with native input.""" + if not isinstance(token, _FrameCaptureCut): + raise InputObserverError("X RECORD received an invalid frame-boundary token") + with self._frame_cut_state_lock: + if self._frame_cut is not None: + raise InputObserverError("an X RECORD frame marker is already pending") + self._frame_cut = token + try: + self._send_frame_cut_marker() + if not token.marker_seen.wait(timeout=self.shutdown_timeout): + raise InputObserverError( + "X RECORD did not deliver the post-capture frame marker " + f"within {self.shutdown_timeout:.1f}s" + ) + self.check_health() + with self._frame_cut_state_lock: + return self._device_event_count == token.start_device_count + except BaseException as exc: + self.complete_frame_capture(token) + self._fail(exc) + raise + + def complete_frame_capture(self, token: object | None) -> None: + """Let X RECORD process events after the frame commit boundary.""" + if not isinstance(token, _FrameCaptureCut): + return + release_serial = False + with self._frame_cut_state_lock: + if self._frame_cut is token: + self._frame_cut = None + release_serial = True + elif not token.release_marker.is_set(): + release_serial = True + token.release_marker.set() + if release_serial: + self._frame_cut_serial.release() + def _handle_device_event(self, event: _CoreWireEvent) -> None: """Accept the next event in the global device stream.""" + with self._frame_cut_state_lock: + self._device_event_count += 1 self._finalize_pending() observed_at = time.time() if event.event_type == _MOTION_NOTIFY: @@ -828,14 +937,18 @@ def _handle_device_event(self, event: _CoreWireEvent) -> None: position = (float(event.root_x), float(event.root_y)) self._last_pointer = position if self.capture_mouse_moves: - self._emit( - ObservedMouseMove( - x=position[0], - y=position[1], - injected=event.injected, - timestamp=observed_at, - ) + receipt = ( + None + if event.injected + else self._reserve_receipt(observed_at) ) + observed = ObservedMouseMove( + x=position[0], + y=position[1], + injected=event.injected, + timestamp=observed_at, + ) + self._emit_received(observed, receipt) return if event.event_type in {_KEY_PRESS, _KEY_RELEASE}: if not self.observe_keyboard: @@ -845,6 +958,21 @@ def _handle_device_event(self, event: _CoreWireEvent) -> None: return else: # pragma: no cover - guarded by the callback range check return + recordable_button = not ( + event.event_type == _BUTTON_RELEASE and event.detail in {4, 5, 6, 7} + ) and not ( + event.event_type in {_BUTTON_PRESS, _BUTTON_RELEASE} + and event.detail <= 0 + ) + receipt = ( + self._reserve_receipt(observed_at) + if not event.injected + and ( + event.event_type in {_KEY_PRESS, _KEY_RELEASE} + or recordable_button + ) + else None + ) self._pending = _PendingDeviceEvent( event_type=event.event_type, detail=event.detail, @@ -852,6 +980,7 @@ def _handle_device_event(self, event: _CoreWireEvent) -> None: injected=event.injected, receipt_timestamp=observed_at, deadline=time.monotonic() + _CORRELATION_TIMEOUT_SECONDS, + receipt=receipt, ) def _handle_delivered_event( @@ -913,38 +1042,36 @@ def _finalize_pending(self) -> None: self._text_state_uncertain = True if self._compose_state is not None: self._xkbcommon.xkb_compose_state_reset(self._compose_state) - self._emit( - normalize_xinput_key_event( - keycode=pending.detail, - pressed=pressed, - keysym_name=None, - injected=injected, - character=None, - derive_character=False, - timestamp=pending.receipt_timestamp, - ) + observed = normalize_xinput_key_event( + keycode=pending.detail, + pressed=pressed, + keysym_name=None, + injected=injected, + character=None, + derive_character=False, + timestamp=pending.receipt_timestamp, ) + self._emit_received(observed, pending.receipt) return keysym, keysym_name = self._lookup_keysym( pending.detail, candidate.state, ) - self._emit( - normalize_xinput_key_event( + observed = normalize_xinput_key_event( + keycode=pending.detail, + pressed=pressed, + keysym_name=keysym_name, + injected=injected, + character=self._resolved_character( keycode=pending.detail, - pressed=pressed, + keysym=keysym, keysym_name=keysym_name, - injected=injected, - character=self._resolved_character( - keycode=pending.detail, - keysym=keysym, - keysym_name=keysym_name, - pressed=pressed, - ), - derive_character=False, - timestamp=pending.receipt_timestamp, - ) + pressed=pressed, + ), + derive_character=False, + timestamp=pending.receipt_timestamp, ) + self._emit_received(observed, pending.receipt) return if candidate is not None: @@ -966,7 +1093,14 @@ def _finalize_pending(self) -> None: timestamp=pending.receipt_timestamp, ) if event is not None: - self._emit(event) + self._emit_received(event, pending.receipt) + elif pending.receipt is not None: + failure = InputObserverError( + "X RECORD reserved an input receipt that did not normalize " + "to a recordable event" + ) + self._fail_receipt(pending.receipt, failure) + raise failure def _finalize_expired_pending(self) -> None: pending = self._pending @@ -1260,6 +1394,15 @@ def attempt(label: str, operation): self._baseline_marker_seen = False self._control_id_base = 0 self._root = 0 + if self._pending is not None and self._pending.receipt is not None: + failure = ( + cleanup_failures[0] + if cleanup_failures + else InputObserverError( + "X RECORD stopped before a reserved input was delivered" + ) + ) + self._fail_receipt(self._pending.receipt, failure) self._pending = None self._delivered_correlation_uncertain = False self._last_pointer = None @@ -1269,6 +1412,7 @@ def attempt(label: str, operation): self._unverifiable_keycodes.clear() if cleanup_failures: primary = cleanup_failures[0] + self._fail_unqueued_receipts(primary) for secondary in cleanup_failures[1:]: add_exception_note( primary, diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py index 11a4851..036610e 100644 --- a/openadapt_capture/input_observer/windows.py +++ b/openadapt_capture/input_observer/windows.py @@ -151,6 +151,7 @@ class _RawKeyboardTransition: flags: int keyboard_layout: int timestamp: float + receipt: object | None = None @dataclass(frozen=True, slots=True) @@ -163,6 +164,7 @@ class _RawMouseTransition: mouse_data: int flags: int timestamp: float + receipt: object | None = None _RawInputTransition = _RawKeyboardTransition | _RawMouseTransition @@ -823,18 +825,44 @@ def _input_translation_main(self) -> None: else self._translate_mouse(item) ) if event is not None: - self._emit(event) + self._emit_received(event, item.receipt) + elif item.receipt is not None: + raise InputObserverError( + "Windows reserved an input receipt that did not normalize " + "to a recordable event" + ) except BaseException as exc: failure = ( exc if isinstance(exc, InputObserverError) else InputObserverError(f"Windows input translation failed: {exc}") ) + if isinstance( + item, + (_RawKeyboardTransition, _RawMouseTransition), + ): + self._fail_receipt(item.receipt, failure) self._fail(failure) + self._discard_translation_queue(failure) return finally: self._translation_queue.task_done() + def _discard_translation_queue(self, failure: BaseException) -> None: + """Fail receipts left behind by a stopped translation worker.""" + while True: + try: + item = self._translation_queue.get_nowait() + except queue.Empty: + return + else: + if isinstance( + item, + (_RawKeyboardTransition, _RawMouseTransition), + ): + self._fail_receipt(item.receipt, failure) + self._translation_queue.task_done() + def _start_input_translation(self) -> None: thread = self._translation_thread if thread is not None and thread.is_alive(): @@ -888,6 +916,8 @@ def _keyboard_hook_callback( wparam: int, lparam: int, ) -> int: + receipt: object | None = None + callback_entered = False try: if code == HC_ACTION and wparam in { WM_KEYDOWN, @@ -895,13 +925,18 @@ def _keyboard_hook_callback( WM_SYSKEYDOWN, WM_SYSKEYUP, }: + self._begin_native_callback() + callback_entered = True timestamp = self._clock() payload = ctypes.cast( lparam, ctypes.POINTER(KBDLLHOOKSTRUCT), ).contents injected = bool(payload.flags & (LLKHF_INJECTED | LLKHF_LOWER_IL_INJECTED)) - if not injected: + if injected: + self._mark_native_activity() + else: + receipt = self._reserve_receipt(timestamp) keyboard_layout = self._foreground_keyboard_layout() self._enqueue_input( _RawKeyboardTransition( @@ -911,11 +946,20 @@ def _keyboard_hook_callback( flags=int(payload.flags), keyboard_layout=keyboard_layout, timestamp=timestamp, + receipt=receipt, ) ) except BaseException as exc: + self._fail_receipt(receipt, exc) self._record_callback_failure(exc) - return self._call_next(self._keyboard_hook, code, wparam, lparam) + try: + return self._call_next(self._keyboard_hook, code, wparam, lparam) + finally: + if callback_entered: + try: + self._end_native_callback() + except BaseException as exc: + self._record_callback_failure(exc) def _mouse_hook_callback( self, @@ -923,25 +967,55 @@ def _mouse_hook_callback( wparam: int, lparam: int, ) -> int: + receipt: object | None = None + callback_entered = False try: if code == HC_ACTION: - timestamp = self._clock() - payload = ctypes.cast( - lparam, - ctypes.POINTER(MSLLHOOKSTRUCT), - ).contents - injected = bool(payload.flags & (LLMHF_INJECTED | LLMHF_LOWER_IL_INJECTED)) - if not injected and (wparam != WM_MOUSEMOVE or self.capture_mouse_moves): - self._enqueue_input( - _RawMouseTransition( - message=wparam, - x=int(payload.pt.x), - y=int(payload.pt.y), - mouse_data=int(payload.mouseData), - flags=int(payload.flags), - timestamp=timestamp, - ) + observed_message = ( + wparam in _BUTTON_MESSAGES + or wparam in { + WM_XBUTTONDOWN, + WM_XBUTTONUP, + WM_MOUSEWHEEL, + WM_MOUSEHWHEEL, + } + or (wparam == WM_MOUSEMOVE and self.capture_mouse_moves) + ) + if observed_message: + self._begin_native_callback() + callback_entered = True + timestamp = self._clock() + payload = ctypes.cast( + lparam, + ctypes.POINTER(MSLLHOOKSTRUCT), + ).contents + injected = bool( + payload.flags + & (LLMHF_INJECTED | LLMHF_LOWER_IL_INJECTED) ) + if injected: + self._mark_native_activity() + else: + receipt = self._reserve_receipt(timestamp) + self._enqueue_input( + _RawMouseTransition( + message=wparam, + x=int(payload.pt.x), + y=int(payload.pt.y), + mouse_data=int(payload.mouseData), + flags=int(payload.flags), + timestamp=timestamp, + receipt=receipt, + ) + ) except BaseException as exc: + self._fail_receipt(receipt, exc) self._record_callback_failure(exc) - return self._call_next(self._mouse_hook, code, wparam, lparam) + try: + return self._call_next(self._mouse_hook, code, wparam, lparam) + finally: + if callback_entered: + try: + self._end_native_callback() + except BaseException as exc: + self._record_callback_failure(exc) diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index 1ae7034..186857f 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -91,15 +91,20 @@ def _first_structural_observation( def _merged_frame_binding(events: list[ActionEvent]) -> dict[str, float | int | None]: - """Return the terminal child binding and reject a mixed native epoch.""" + """Return the aggregate's initial before-frame and reject a mixed epoch.""" if not events: return { "source_ordinal": None, "screenshot_timestamp": None, "screenshot_source_ordinal": None, + "after_screenshot_timestamp": None, + "after_screenshot_source_ordinal": None, "window_event_timestamp": None, "window_event_source_ordinal": None, + "after_window_event_timestamp": None, + "after_window_event_source_ordinal": None, "window_geometry_generation": None, + "after_window_geometry_generation": None, } generations = [event.window_geometry_generation for event in events] if any(value is not None for value in generations): @@ -115,23 +120,38 @@ def _merged_frame_binding(events: list[ActionEvent]) -> dict[str, float | int | if ( event.screenshot_timestamp is None or event.screenshot_source_ordinal is None + or event.after_screenshot_timestamp is None + or event.after_screenshot_source_ordinal is None or event.window_event_timestamp is None or event.window_event_source_ordinal is None + or event.after_window_event_timestamp is None + or event.after_window_event_source_ordinal is None + or event.after_window_geometry_generation is None or event.screenshot_timestamp != event.window_event_timestamp or event.screenshot_source_ordinal != event.window_event_source_ordinal + or event.after_screenshot_timestamp + != event.after_window_event_timestamp + or event.after_screenshot_source_ordinal + != event.after_window_event_source_ordinal ): raise ValueError( - "cannot merge a native action without one atomic frame/window pair" + "cannot merge a native action without atomic before/after pairs" ) + initial = events[0] terminal = events[-1] return { "source_ordinal": terminal.source_ordinal, - "screenshot_timestamp": terminal.screenshot_timestamp, - "screenshot_source_ordinal": terminal.screenshot_source_ordinal, - "window_event_timestamp": terminal.window_event_timestamp, - "window_event_source_ordinal": terminal.window_event_source_ordinal, - "window_geometry_generation": terminal.window_geometry_generation, + "screenshot_timestamp": initial.screenshot_timestamp, + "screenshot_source_ordinal": initial.screenshot_source_ordinal, + "after_screenshot_timestamp": terminal.after_screenshot_timestamp, + "after_screenshot_source_ordinal": terminal.after_screenshot_source_ordinal, + "window_event_timestamp": initial.window_event_timestamp, + "window_event_source_ordinal": initial.window_event_source_ordinal, + "after_window_event_timestamp": terminal.after_window_event_timestamp, + "after_window_event_source_ordinal": terminal.after_window_event_source_ordinal, + "window_geometry_generation": initial.window_geometry_generation, + "after_window_geometry_generation": terminal.after_window_geometry_generation, } @@ -473,58 +493,76 @@ def merge_consecutive_mouse_click_events( def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: return ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5 - # Build timestamp mappings for down -> up and down -> next_down (for double-click) - down_events: list[MouseDownEvent] = [] - down_to_up: dict[float, MouseUpEvent] = {} - down_to_next_down: dict[float, MouseDownEvent] = {} + # Use capture order for identity. Wall-clock timestamps can repeat or reset; + # they remain valid only for interval calculations. + def event_identity(event: ActionEvent, index: int) -> tuple[str, int]: + if event.source_ordinal is not None: + return "source", event.source_ordinal + return "index", index + + identities = [event_identity(event, index) for index, event in enumerate(events)] + source_identities = [identity for identity in identities if identity[0] == "source"] + if len(source_identities) != len(set(source_identities)): + raise ValueError("cannot merge actions with duplicate source ordinals") + + down_events: list[tuple[tuple[str, int], MouseDownEvent]] = [] + down_to_up: dict[ + tuple[str, int], tuple[tuple[str, int], MouseUpEvent] + ] = {} + down_to_next_down: dict[ + tuple[str, int], tuple[tuple[str, int], MouseDownEvent] + ] = {} # First pass: collect all down events and map to their up events - prev_down: MouseDownEvent | None = None - for event in events: + prev_down: tuple[tuple[str, int], MouseDownEvent] | None = None + for event_index, event in enumerate(events): + identity = identities[event_index] if isinstance(event, MouseDownEvent): - down_events.append(event) + down_events.append((identity, event)) # Check if this could be second click of a double-click if prev_down is not None: - dt = event.timestamp - prev_down.timestamp - dx = abs(event.x - prev_down.x) - dy = abs(event.y - prev_down.y) + prev_identity, prev_event = prev_down + dt = event.timestamp - prev_event.timestamp + dx = abs(event.x - prev_event.x) + dy = abs(event.y - prev_event.y) if ( dt <= double_click_interval and dx <= double_click_distance and dy <= double_click_distance - and event.button == prev_down.button + and event.button == prev_event.button ): - down_to_next_down[prev_down.timestamp] = event - prev_down = event + down_to_next_down[prev_identity] = (identity, event) + prev_down = (identity, event) elif isinstance(event, MouseUpEvent): # Find the most recent unmatched down with same button - for down in reversed(down_events): - if down.button == event.button and down.timestamp not in down_to_up: + for down_identity, down in reversed(down_events): + if down.button == event.button and down_identity not in down_to_up: # Only map if distance is small enough (not a drag) dist = calculate_distance(down.x, down.y, event.x, event.y) if dist <= drag_distance_threshold: - down_to_up[down.timestamp] = event + down_to_up[down_identity] = (identity, event) break # Second pass: generate merged events result = [] - skip_timestamps: set[float] = set() + skip_identities: set[tuple[str, int]] = set() - for event in events: - if event.timestamp in skip_timestamps: + for event_index, event in enumerate(events): + identity = identities[event_index] + if identity in skip_identities: continue if isinstance(event, MouseDownEvent): down = event - if down.timestamp in down_to_up: - up = down_to_up[down.timestamp] + if identity in down_to_up: + up_identity, up = down_to_up[identity] # Check if this is the start of a double-click - if down.timestamp in down_to_next_down: - next_down = down_to_next_down[down.timestamp] - if next_down.timestamp in down_to_up: - next_up = down_to_up[next_down.timestamp] + if identity in down_to_next_down: + next_down_identity, next_down = down_to_next_down[identity] + if next_down_identity in down_to_up: + next_up_identity, next_up = down_to_up[next_down_identity] # Create double-click double_click = MouseDoubleClickEvent( @@ -539,9 +577,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: **_merged_frame_binding([down, up, next_down, next_up]), ) result.append(double_click) - skip_timestamps.add(up.timestamp) - skip_timestamps.add(next_down.timestamp) - skip_timestamps.add(next_up.timestamp) + skip_identities.add(up_identity) + skip_identities.add(next_down_identity) + skip_identities.add(next_up_identity) continue # Create single click @@ -557,7 +595,7 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: **_merged_frame_binding([down, up]), ) result.append(single_click) - skip_timestamps.add(up.timestamp) + skip_identities.add(up_identity) else: # Unmatched down event result.append(event) diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 94ad115..2bd32f1 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -51,6 +51,8 @@ from openadapt_capture.desktop_capture import DesktopCaptureScope from openadapt_capture.extensions import synchronized_queue as sq from openadapt_capture.input_observer import ( + InputObserver, + InputObserverError, ObservedInput, ObservedKey, ObservedMouseButton, @@ -83,6 +85,93 @@ class WindowScopedFrame: geometry_generation: int +@dataclass(frozen=True) +class _NativeFrameBoundaryUse: + observer: InputObserver + token: object | None + + +class _NativeFrameBoundaryClosed(RuntimeError): + """The input reader closed before another ordinary frame could start.""" + + +class NativeInputFrameBoundary: + """Coordinate window pixels with the active native input observer.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._observer: InputObserver | None = None + self._state = "pending" + self._failure: BaseException | None = None + self._active = 0 + + def attach(self, observer: InputObserver) -> None: + with self._condition: + if self._state != "pending" or self._observer is not None: + raise InputObserverError("native frame boundary attached more than once") + self._observer = observer + self._state = "active" + self._condition.notify_all() + + def begin(self) -> _NativeFrameBoundaryUse: + with self._condition: + while self._state == "pending": + self._condition.wait() + if self._failure is not None: + raise self._failure + if self._state != "active" or self._observer is None: + raise _NativeFrameBoundaryClosed() + observer = self._observer + self._active += 1 + try: + token = observer.begin_frame_capture() + except BaseException: + with self._condition: + self._active -= 1 + self._condition.notify_all() + raise + return _NativeFrameBoundaryUse(observer, token) + + @staticmethod + def finish(use: _NativeFrameBoundaryUse) -> bool: + return use.observer.finish_frame_capture(use.token) + + @staticmethod + def seal(use: _NativeFrameBoundaryUse) -> None: + use.observer.seal_frame_capture(use.token) + + def complete(self, use: _NativeFrameBoundaryUse) -> None: + try: + use.observer.complete_frame_capture(use.token) + finally: + with self._condition: + self._active -= 1 + self._condition.notify_all() + + def begin_close(self) -> None: + with self._condition: + if self._state in {"closed", "failed"}: + return + self._state = "closing" + self._condition.notify_all() + while self._active: + self._condition.wait() + + def close(self) -> None: + with self._condition: + if self._state == "failed": + return + self._state = "closed" + self._condition.notify_all() + + def fail(self, failure: BaseException) -> None: + with self._condition: + if self._failure is None: + self._failure = failure + self._state = "failed" + self._condition.notify_all() + + try: import soundfile except ImportError: @@ -155,6 +244,11 @@ def __init__(self, journal: "OrderedEventJournal", entry: _JournalEntry) -> None def source_ordinal(self) -> int: return self._entry.sequence + @property + def finished(self) -> bool: + """Return whether this producer has completed or failed its position.""" + return self._finished + def complete(self, event: Event) -> None: if self._finished: raise RuntimeError("the event journal reservation is already complete") @@ -182,6 +276,48 @@ def fail(self, error: BaseException) -> None: self._journal._condition.notify_all() +class WindowActionReservation: + """A source position and the exact published geometry at native receipt.""" + + def __init__( + self, + reservation: EventReservation, + window_scope: WindowCaptureScope, + geometry: tuple[Any, float, float, tuple[int, int, int, int], int], + ) -> None: + self._reservation = reservation + self._window_scope = window_scope + self._geometry = geometry + + @property + def source_ordinal(self) -> int: + return self._reservation.source_ordinal + + @property + def finished(self) -> bool: + return self._reservation.finished + + def bind( + self, + x: float | None, + y: float | None, + ) -> tuple[float, float, int] | int: + """Bind normalized input to geometry reserved at native receipt.""" + if x is not None and y is not None: + return self._window_scope.translate_reserved_geometry( + self._geometry, + x, + y, + ) + return self._window_scope.generation_for_reserved_geometry(self._geometry) + + def complete(self, event: Event) -> None: + self._reservation.complete(event) + + def fail(self, error: BaseException) -> None: + self._reservation.fail(error) + + class OrderedEventJournal: """A causal FIFO journal with pre-observation reservations.""" @@ -241,6 +377,29 @@ def reserve_window_action( raise return reservation, binding + def reserve_window_action_receipt( + self, + timestamp: float, + window_scope: WindowCaptureScope, + ) -> WindowActionReservation: + """Reserve order and geometry at the first native input boundary.""" + timestamp = float(timestamp) + if not math.isfinite(timestamp): + raise EventJournalOrderingError("event timestamps must be finite") + with window_scope.observation_boundary(): + with self._condition: + entry = self._reserve_locked(timestamp) + reservation = EventReservation(self, entry) + try: + geometry = window_scope.reserve_action_geometry() + except BaseException as exc: + entry.error = exc + entry.ready = True + reservation._finished = True + self._condition.notify_all() + raise + return WindowActionReservation(reservation, window_scope, geometry) + def put(self, event: Event, block: bool = True, timeout: float | None = None) -> None: del block, timeout reservation = self.reserve(event.timestamp) @@ -364,6 +523,7 @@ def __bool__(self): NUM_MEMORY_STATS_TO_LOG = 3 STARTUP_WAIT_POLL_SECONDS = 0.1 PRE_READY_TASK_JOIN_TIMEOUT_SECONDS = 2.0 +TERMINAL_FRAME_SEAL_TIMEOUT_SECONDS = 10.0 stop_sequence_detected = False @@ -588,6 +748,7 @@ def process_events( prev_saved_window_timestamp = 0 prev_saved_screen_ordinal = 0 prev_saved_window_ordinal = 0 + pending_action_events: list[Event] = [] started = False def processing_complete() -> bool: @@ -595,6 +756,53 @@ def processing_complete() -> bool: return producers_finished.is_set() and event_q.empty() return terminate_processing.is_set() and event_q.empty() + def write_bound_action(action_event: Event) -> None: + process_event( + action_event, + action_write_q, + write_action_event, + recording, + perf_q, + ) + num_action_events.value += 1 + + def bind_pending_actions( + after_screen_event: Event, + after_window_event: Event | None, + ) -> None: + """Bind pending actions to this first ordinal-later retained frame.""" + for action_event in pending_action_events: + action_event.data["after_screenshot_timestamp"] = after_screen_event.timestamp + action_event.data["after_screenshot_source_ordinal"] = ( + after_screen_event.source_ordinal + ) + action_generation = action_event.data.get("window_geometry_generation") + if after_window_event is not None: + action_event.data["after_window_event_timestamp"] = ( + after_window_event.timestamp + ) + action_event.data["after_window_event_source_ordinal"] = ( + after_window_event.source_ordinal + ) + after_generation = after_window_event.data.get("state", {}).get( + "geometry_generation" + ) + action_event.data["after_window_geometry_generation"] = after_generation + if ( + after_window_event.timestamp != after_screen_event.timestamp + or after_window_event.source_ordinal + != after_screen_event.source_ordinal + ): + raise WindowCaptureError( + "the action after frame and native geometry are not one atomic pair" + ) + elif action_generation is not None: + raise WindowCaptureError( + "a native action has no geometry paired with its after frame" + ) + write_bound_action(action_event) + pending_action_events.clear() + while not processing_complete(): # Bounded get: a bare event_q.get() deadlocks shutdown when terminate # is set while the queue is empty and the readers have already exited @@ -634,6 +842,7 @@ def processing_complete() -> bool: # XXX TODO: mitigate if event.type == "screen": scoped_pair = False + current_window_event = None if isinstance(event.data, WindowScopedFrame): scoped_pair = True scoped_frame = event.data @@ -644,13 +853,15 @@ def processing_complete() -> bool: raise WindowCaptureError( "the scoped frame geometry generation differs from its metadata" ) - prev_window_event = Event( + current_window_event = Event( event.timestamp, "window", scoped_frame.window_event_data, event.source_ordinal, ) + prev_window_event = current_window_event event = event._replace(data=scoped_frame.image) + bind_pending_actions(event, current_window_event) prev_screen_event = event if config.RECORD_FULL_VIDEO: video_event = event._replace(type="screen/video") @@ -739,15 +950,7 @@ def processing_complete() -> bool: "the action frame and native geometry have different source ordinals" ) - process_event( - event, - action_write_q, - write_action_event, - recording, - perf_q, - ) - - num_action_events.value += 1 + pending_action_events.append(event) screen_is_new = ( prev_screen_event.source_ordinal > prev_saved_screen_ordinal @@ -800,6 +1003,16 @@ def processing_complete() -> bool: raise Exception(f"unhandled {event.type=}") del prev_event prev_event = event + if pending_action_events: + if any( + event.data.get("window_geometry_generation") is not None + for event in pending_action_events + ): + raise WindowCaptureError( + "native recording ended before pending actions received an after frame" + ) + for event in pending_action_events: + write_bound_action(event) logger.info("Done") @@ -1147,6 +1360,7 @@ def trigger_action_event( coordinate_scope: CoordinateScope | None = None, timestamp: float | None = None, structural_observer: StructuralObserver | None = None, + reservation: EventReservation | WindowActionReservation | None = None, ) -> None: """Triggers an action event and adds it to the event queue. @@ -1161,6 +1375,7 @@ def trigger_action_event( clock only for legacy/direct callers. structural_observer: Optional accessibility observer. Evidence is captured against global coordinates before any window translation. + reservation: Optional source position created at native input receipt. Returns: None @@ -1170,7 +1385,22 @@ def trigger_action_event( x = event_data.get("mouse_x") y = event_data.get("mouse_y") window_binding: tuple[float, float, int] | int | None = None - if isinstance(event_q, OrderedEventJournal) and isinstance( + if reservation is not None: + try: + if isinstance(coordinate_scope, WindowCaptureScope): + if not isinstance(reservation, WindowActionReservation): + raise EventJournalOrderingError( + "window-scoped input requires a receipt-time geometry reservation" + ) + window_binding = reservation.bind(x, y) + elif not isinstance(reservation, EventReservation): + raise EventJournalOrderingError( + "unscoped input received an incompatible source reservation" + ) + except BaseException as exc: + reservation.fail(exc) + raise + elif isinstance(event_q, OrderedEventJournal) and isinstance( coordinate_scope, WindowCaptureScope, ): @@ -1247,6 +1477,7 @@ def on_move( y: float, injected: bool = False, timestamp: float | None = None, + reservation: EventReservation | WindowActionReservation | None = None, ) -> None: """Handles the 'move' event. @@ -1267,6 +1498,7 @@ def on_move( {"name": "move", "mouse_x": x, "mouse_y": y}, coordinate_scope, timestamp, + reservation=reservation, ) @@ -1280,6 +1512,7 @@ def on_click( injected: bool = False, timestamp: float | None = None, structural_observer: StructuralObserver | None = None, + reservation: EventReservation | WindowActionReservation | None = None, ) -> None: """Handles the 'click' event. @@ -1309,6 +1542,7 @@ def on_click( coordinate_scope, timestamp, structural_observer if pressed else None, + reservation, ) @@ -1322,6 +1556,7 @@ def on_scroll( injected: bool = False, timestamp: float | None = None, structural_observer: StructuralObserver | None = None, + reservation: EventReservation | WindowActionReservation | None = None, ) -> None: """Handles the 'scroll' event. @@ -1351,6 +1586,7 @@ def on_scroll( coordinate_scope, timestamp, structural_observer, + reservation, ) @@ -1359,6 +1595,7 @@ def handle_key( key: ObservedKey, coordinate_scope: CoordinateScope | None = None, structural_observer: StructuralObserver | None = None, + reservation: EventReservation | WindowActionReservation | None = None, ) -> None: """Persist a normalized native key transition. @@ -1383,6 +1620,7 @@ def handle_key( coordinate_scope=coordinate_scope, timestamp=key.timestamp, structural_observer=structural_observer if key.pressed else None, + reservation=reservation, ) @@ -1395,6 +1633,8 @@ def read_screen_events( window_scope: WindowCaptureScope | None = None, desktop_scope: DesktopCaptureScope | None = None, input_finished: threading.Event | None = None, + input_frame_boundary: NativeInputFrameBoundary | None = None, + terminal_frame_finished: threading.Event | None = None, ) -> None: """Read screen events and add them to the event queue. @@ -1406,6 +1646,8 @@ def read_screen_events( re-resolved every frame (windows move/resize) and a bounds-timeline "window" event is queued whenever the resolved bounds change, so converters can reconstruct the exact window position for every action. + Both native scopes reject frames crossed by input and seal one clean + terminal frame after the last accepted action. Args: event_q: A queue for adding screen events. @@ -1416,8 +1658,11 @@ def read_screen_events( window_scope: Optional window scope for window-pixel-space capture. desktop_scope: Full-screen virtual-desktop contract. It verifies the monitor topology before and after each captured frame. - input_finished: Input-reader completion boundary. Window capture waits - for it before it records the terminal after-action frame. + input_finished: Input-reader completion boundary. The terminal native + frame waits for observer shutdown after it seals native input. + input_frame_boundary: Active observer bridge for input-stable frames. + terminal_frame_finished: Signals that the exact terminal frame sealed + native input and entered the ordered journal. """ if window_scope is not None and desktop_scope is not None: raise ValueError("screen reader cannot use both window and desktop scopes") @@ -1429,10 +1674,23 @@ def read_screen_events( logger.info(f"Starting (fps={fps}, min_interval={min_interval:.3f}s)") started = False - def capture_one() -> tuple[float, float]: + def capture_one( + *, + require_input_boundary: bool = True, + seal_input: bool = False, + ) -> tuple[float, float] | None: nonlocal started t_start = time.perf_counter() - if window_scope is not None: + terminal_deadline = ( + time.monotonic() + TERMINAL_FRAME_SEAL_TIMEOUT_SECONDS + if seal_input + else None + ) + if window_scope is not None or desktop_scope is not None: + if seal_input and input_frame_boundary is None: + raise WindowCaptureError( + "terminal native capture requires the native input boundary" + ) # Do not hold the observation boundary during pixel acquisition. # An OS input callback that arrives while the grab is in flight must # reserve and bind the previously published frame before this new @@ -1443,37 +1701,80 @@ def capture_one() -> tuple[float, float]: # Any failed capture terminates the session. Retrying would omit a # frame while input continues and could produce complete-looking # evidence with a missing interval. - screenshot, _window_changed = window_scope.capture_frame(publish=False) - t_screenshot = time.perf_counter() - if screenshot is None: - raise WindowCaptureError("the captured screenshot was empty") - if not started: - started_event.set() - started = True - frame_timestamp = utils.get_timestamp() - if not isinstance(event_q, OrderedEventJournal): - raise WindowCaptureError("window-scoped capture requires the ordered event journal") - generation = window_scope.current_generation() - scoped_frame = WindowScopedFrame( - image=screenshot, - window_event_data=window_scope.window_event_data(), - geometry_generation=generation, - ) - event_q.commit_window_frame( - Event(frame_timestamp, "screen", scoped_frame), - window_scope, - generation, - ) - return t_start, t_screenshot - if desktop_scope is not None: - # A monitor can move or change scale while the combined frame keeps - # the same dimensions. Check both sides of the grab so neither the - # frame nor later input uses stale origin or monitor geometry. - desktop_scope.assert_current(force=True) - screenshot = utils.take_screenshot() - desktop_scope.assert_current(force=True) - else: - screenshot = utils.take_screenshot() + while True: + boundary_use = None + if input_frame_boundary is not None and require_input_boundary: + try: + boundary_use = input_frame_boundary.begin() + except _NativeFrameBoundaryClosed: + return None + try: + if window_scope is not None: + screenshot, _window_changed = window_scope.capture_frame( + publish=False + ) + else: + assert desktop_scope is not None + # A monitor can move or change scale while the combined + # frame keeps the same dimensions. Check both sides of + # the grab so the pixels and input use one topology. + desktop_scope.assert_current(force=True) + screenshot = utils.take_screenshot() + desktop_scope.assert_current(force=True) + t_screenshot = time.perf_counter() + if screenshot is None: + raise WindowCaptureError("the captured screenshot was empty") + frame_timestamp = utils.get_timestamp() + if boundary_use is not None and not input_frame_boundary.finish( + boundary_use + ): + input_frame_boundary.complete(boundary_use) + boundary_use = None + if terminate_processing.is_set() and not seal_input: + return None + if ( + terminal_deadline is not None + and time.monotonic() >= terminal_deadline + ): + raise WindowCaptureError( + "native input did not become stable before the " + "terminal-frame deadline" + ) + if min_interval > 0: + if seal_input: + remaining = terminal_deadline - time.monotonic() + time.sleep(min(min_interval, max(0.0, remaining))) + else: + terminate_processing.wait(min_interval) + continue + if boundary_use is not None and seal_input: + input_frame_boundary.seal(boundary_use) + if not isinstance(event_q, OrderedEventJournal): + raise WindowCaptureError( + "native-scoped capture requires the ordered event journal" + ) + if window_scope is not None: + generation = window_scope.current_generation() + scoped_frame = WindowScopedFrame( + image=screenshot, + window_event_data=window_scope.window_event_data(), + geometry_generation=generation, + ) + event_q.commit_window_frame( + Event(frame_timestamp, "screen", scoped_frame), + window_scope, + generation, + ) + else: + event_q.put(Event(frame_timestamp, "screen", screenshot)) + if not started: + started_event.set() + started = True + return t_start, t_screenshot + finally: + if boundary_use is not None: + input_frame_boundary.complete(boundary_use) + screenshot = utils.take_screenshot() t_screenshot = time.perf_counter() if screenshot is None: raise WindowCaptureError("the captured screenshot was empty") @@ -1485,7 +1786,10 @@ def capture_one() -> tuple[float, float]: return t_start, t_screenshot while not terminate_processing.is_set(): - t_start, t_screenshot = capture_one() + timing = capture_one() + if timing is None: + break + t_start, t_screenshot = timing # Throttle: sleep for the remainder of the frame interval if min_interval > 0: elapsed = time.perf_counter() - t_start @@ -1496,10 +1800,26 @@ def capture_one() -> tuple[float, float]: t_end = time.perf_counter() _screen_timing.append((t_screenshot - t_start, t_end - t_start)) - if window_scope is not None and input_finished is not None: + if (window_scope is not None or desktop_scope is not None) and ( + terminal_frame_finished is not None + ): + timing = capture_one(seal_input=True) + if timing is None: + raise WindowCaptureError("the terminal native frame was not committed") + terminal_frame_finished.set() + if input_finished is not None: + input_finished.wait() + if _screen_timing is not None and timing is not None: + t_start, t_screenshot = timing + t_end = time.perf_counter() + _screen_timing.append((t_screenshot - t_start, t_end - t_start)) + elif (window_scope is not None or desktop_scope is not None) and ( + input_finished is not None + ): input_finished.wait() - t_start, t_screenshot = capture_one() - if _screen_timing is not None: + timing = capture_one(require_input_boundary=False) + if _screen_timing is not None and timing is not None: + t_start, t_screenshot = timing t_end = time.perf_counter() _screen_timing.append((t_screenshot - t_start, t_end - t_start)) logger.info("Done") @@ -1742,12 +2062,17 @@ def read_input_events( coordinate_scope: CoordinateScope | None = None, structural_observer: StructuralObserver | None = None, finished_event: threading.Event | None = None, + input_frame_boundary: NativeInputFrameBoundary | None = None, + terminal_frame_finished: threading.Event | None = None, ) -> None: """Read globally ordered keyboard and mouse events from one native observer.""" stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] stop_sequence_indices = [0 for _ in stop_sequences] - def on_observed(event: ObservedInput) -> None: + def on_observed( + event: ObservedInput, + reservation: EventReservation | WindowActionReservation | None = None, + ) -> None: if isinstance(event, ObservedMouseMove): on_move( event_q, @@ -1756,6 +2081,7 @@ def on_observed(event: ObservedInput) -> None: event.y, event.injected, timestamp=event.timestamp, + reservation=reservation, ) return if isinstance(event, ObservedMouseButton): @@ -1769,6 +2095,7 @@ def on_observed(event: ObservedInput) -> None: event.injected, timestamp=event.timestamp, structural_observer=structural_observer, + reservation=reservation, ) return if isinstance(event, ObservedMouseScroll): @@ -1782,13 +2109,20 @@ def on_observed(event: ObservedInput) -> None: event.injected, timestamp=event.timestamp, structural_observer=structural_observer, + reservation=reservation, ) return if event.injected: return logger.debug(f"{event=}") - handle_key(event_q, event, coordinate_scope, structural_observer) + handle_key( + event_q, + event, + coordinate_scope, + structural_observer, + reservation, + ) if not event.pressed: return @@ -1818,9 +2152,33 @@ def on_observed(event: ObservedInput) -> None: if callable(stop_hook): setattr(on_observed, "_openadapt_delivery_thread_stop", stop_hook) + if isinstance(event_q, OrderedEventJournal): + + def reserve_observed(timestamp: float): + if isinstance(coordinate_scope, WindowCaptureScope): + return event_q.reserve_window_action_receipt( + timestamp, + coordinate_scope, + ) + return event_q.reserve(timestamp) + + def deliver_observed(event: ObservedInput, reservation: object) -> None: + if not isinstance( + reservation, + (EventReservation, WindowActionReservation), + ): + raise EventJournalOrderingError( + "native input delivery received an invalid source reservation" + ) + on_observed(event, reservation) + + setattr(on_observed, "_openadapt_input_receipt", reserve_observed) + setattr(on_observed, "_openadapt_input_delivery", deliver_observed) + utils.set_start_time(recording.timestamp) observer = None started = False + observer_failed = False try: observer = create_input_observer( on_observed, @@ -1830,15 +2188,44 @@ def on_observed(event: ObservedInput) -> None: ) observer.start() started = True + if input_frame_boundary is not None: + input_frame_boundary.attach(observer) started_event.set() while not terminate_processing.wait(0.1): observer.check_health() - except BaseException: + except BaseException as exc: + observer_failed = True + if input_frame_boundary is not None: + input_frame_boundary.fail(exc) terminate_processing.set() raise finally: if started and observer is not None: - observer.stop() + terminal_error = None + if terminal_frame_finished is not None and not observer_failed: + terminal_timeout = max( + 10.0, + float(getattr(observer, "shutdown_timeout", 5.0)) * 2, + ) + if not terminal_frame_finished.wait(timeout=terminal_timeout): + terminal_error = InputObserverError( + "the terminal frame did not seal before native input shutdown" + ) + if input_frame_boundary is not None: + input_frame_boundary.fail(terminal_error) + if input_frame_boundary is not None: + input_frame_boundary.begin_close() + try: + observer.stop() + except BaseException as exc: + if input_frame_boundary is not None: + input_frame_boundary.fail(exc) + raise + else: + if input_frame_boundary is not None: + input_frame_boundary.close() + if terminal_error is not None: + raise terminal_error if finished_event is not None: finished_event.set() @@ -2126,6 +2513,8 @@ def record( event_q = OrderedEventJournal() producers_finished = threading.Event() input_finished = threading.Event() + terminal_frame_finished = threading.Event() + input_frame_boundary = NativeInputFrameBoundary() if window_scope is not None: # The preflight frame sizes the fixed stream. Capture again after the # recording clock starts, then publish pixels and geometry atomically @@ -2146,6 +2535,23 @@ def record( window_scope, initial_generation, ) + else: + assert desktop_scope is not None + # Publish one clean before-frame before the native observer can accept + # input. The screen thread attaches to the observer boundary for every + # later frame, but it cannot safely win that startup race by itself. + desktop_scope.assert_current(force=True) + initial_desktop_frame = utils.take_screenshot() + desktop_scope.assert_current(force=True) + if initial_desktop_frame is None: + raise WindowCaptureError("the initial desktop screenshot was empty") + event_q.put( + Event( + utils.get_timestamp(), + "screen", + initial_desktop_frame, + ) + ) screen_write_q = sq.SynchronizedQueue() action_write_q = sq.SynchronizedQueue() window_write_q = sq.SynchronizedQueue() @@ -2199,6 +2605,8 @@ def record( window_scope, desktop_scope, input_finished, + input_frame_boundary, + terminal_frame_finished, ), terminate_processing, task_errors, @@ -2215,6 +2623,8 @@ def record( window_scope or desktop_scope, structural_observer, input_finished, + input_frame_boundary, + terminal_frame_finished, ) input_event_reader = threading.Thread( target=_run_task_fail_loud, @@ -2749,15 +3159,52 @@ def _control_payload(self) -> dict[str, Any]: def _persist_control_state(self) -> None: if not self._control_enabled: return + self._persist_control_payload(self._control_payload()) + + def _persist_control_payload(self, terminal: dict[str, Any]) -> None: + """Persist one explicit non-secret control snapshot.""" from openadapt_capture.control import write_terminal_state - terminal = self._control_payload() + terminal = dict(terminal) # The file location already binds the capture. Do not retain an # absolute local path (which can disclose a user/profile name) in an # artifact that may later enter sanitization and review. terminal.pop("capture_dir", None) write_terminal_state(self.capture_dir, terminal) + def _stage_completed_control_state(self) -> float: + """Write the exact final state that the immutable seal will inventory.""" + with self._control_state_lock: + finalized_at = time.time() + terminal = self._control_payload() + terminal.update( + { + "phase": "complete", + "complete": True, + "integrity_verified": True, + "error_code": None, + "finalized_at": finalized_at, + } + ) + # Final capture metadata exists even when the live control server + # is disabled. Do not publish these values in memory until sealing + # and its post-write verification succeed. + self._persist_control_payload(terminal) + return finalized_at + + def _publish_completed_control_state(self, finalized_at: float) -> None: + """Publish an already sealed state without rewriting its artifact.""" + with self._control_state_lock: + if self._control_phase in {"complete", "failed", "crashed"}: + raise RuntimeError( + "capture control reached a terminal state before sealing completed" + ) + self._control_phase = "complete" + self._control_complete = True + self._control_integrity_verified = True + self._control_error_code = None + self._control_finalized_at = finalized_at + def _transition_control( self, phase: str, @@ -2933,6 +3380,9 @@ def _seal_completed_capture(self) -> None: }, last_source_ordinal=last_source_ordinal or None, ) + from openadapt_capture.capture import CaptureSession + + CaptureSession.validate_sealed(self.capture_dir) def _start_control_server(self) -> None: from pathlib import Path @@ -3029,13 +3479,9 @@ def _run_record(self) -> None: self.check_health() if self._ready_event.is_set(): self._verify_completed_capture() + finalized_at = self._stage_completed_control_state() self._seal_completed_capture() - self._transition_control( - "complete", - complete=True, - integrity_verified=True, - finalized=True, - ) + self._publish_completed_control_state(finalized_at) else: self._transition_control( "failed", @@ -3218,12 +3664,13 @@ def capture(self): return None self.check_health() if self._capture is None: - try: - from openadapt_capture.capture import CaptureSession + from pathlib import Path - self._capture = CaptureSession.load_verified(self.capture_dir) - except FileNotFoundError: + if not (Path(self.capture_dir) / "recording.db").is_file(): return None + from openadapt_capture.capture import CaptureSession + + self._capture = CaptureSession.load_verified(self.capture_dir) return self._capture diff --git a/openadapt_capture/terminal.py b/openadapt_capture/terminal.py index 14da0ee..e317ba1 100644 --- a/openadapt_capture/terminal.py +++ b/openadapt_capture/terminal.py @@ -21,7 +21,6 @@ _MANIFEST_DOMAIN = b"openadapt.capture-artifact-manifest.v1\0" _TERMINAL_DOMAIN = b"openadapt.capture-terminal.v2\0" _EXCLUDED_ARTIFACTS = { - "capture-state.json", ARTIFACT_MANIFEST_FILENAME, CAPTURE_TERMINAL_FILENAME, } @@ -50,7 +49,7 @@ def _safe_relative_path(self) -> "ArtifactRecord": ): raise ValueError("artifact paths must be safe POSIX-relative paths") if path.as_posix() in _EXCLUDED_ARTIFACTS: - raise ValueError("mutable or seal metadata cannot inventory itself") + raise ValueError("seal metadata cannot inventory itself") return self @@ -332,7 +331,10 @@ def _hash_relative_regular_file(root: Path, relative_path: str) -> tuple[int, st def _read_regular_file(path: Path) -> bytes: """Read one stable regular file through the descriptor that was verified.""" - fd, before = _open_stable_regular_file(path) + try: + fd, before = _open_stable_regular_file(path) + except FileNotFoundError as exc: + raise CaptureSealError(f"capture artifact is missing: {path.name}") from exc chunks: list[bytes] = [] size = 0 try: @@ -485,7 +487,8 @@ def seal_capture( terminal = CaptureTerminal.model_validate(payload) terminal_raw = _canonical_json_bytes(terminal.model_dump(mode="json"), newline=True) _write_new_atomic(root / CAPTURE_TERMINAL_FILENAME, terminal_raw) - return terminal + verified_terminal, _ = verify_capture_artifacts(root) + return verified_terminal def verify_capture_artifacts( @@ -559,3 +562,35 @@ def copy_verified_capture( temporary.cleanup() raise return temporary, destination, terminal + + +def copy_verified_database( + capture_dir: str | os.PathLike[str], +) -> tuple[ + tempfile.TemporaryDirectory[str], + Path, + CaptureTerminal, + CaptureArtifactManifest, +]: + """Copy only the sealed database for bounded-space semantic validation.""" + terminal, manifest = verify_capture_artifacts(capture_dir) + database_record = next( + artifact for artifact in manifest.artifacts if artifact.path == "recording.db" + ) + source = Path(capture_dir).resolve() + temporary = tempfile.TemporaryDirectory(prefix="openadapt-capture-database-") + database_path = Path(temporary.name) / "recording.db" + try: + _copy_verified_regular_file( + source, + database_record.path, + database_path, + database_record, + ) + size, digest = _hash_regular_file(database_path) + if (size, digest) != (database_record.size_bytes, database_record.sha256): + raise CaptureSealError("capture database changed during snapshot") + except BaseException: + temporary.cleanup() + raise + return temporary, database_path, terminal, manifest diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index f1d8e6b..9996996 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -520,6 +520,62 @@ def _geometry_for_action( self._assert_display_topology() return window, scale_x, scale_y, content_rect, generation + def reserve_action_geometry( + self, + ) -> tuple[TargetWindow, float, float, tuple[int, int, int, int], int]: + """Snapshot published geometry without blocking a native input hook.""" + with self._lock: + window = self._published_window + scale_x = self._published_scale_x + scale_y = self._published_scale_y + content_rect = self._published_content_rect + generation = self._published_generation + if window is None or scale_x is None or scale_y is None or content_rect is None: + raise WindowCaptureError( + "an action arrived before the first published frame; " + "capture_frame() must succeed before input can be scoped" + ) + return window, scale_x, scale_y, content_rect, generation + + def _assert_reserved_geometry_current( + self, + geometry: tuple[TargetWindow, float, float, tuple[int, int, int, int], int], + ) -> None: + """Refuse if delivery-time state no longer matches receipt-time state.""" + reserved_window = geometry[0] + self._assert_display_topology() + live = self.resolve() + self._assert_bound_identity(live) + if live.bounds != reserved_window.bounds: + raise WindowCaptureError( + "the target moved or resized after native input receipt; " + "the delayed input cannot be bound to its reserved frame" + ) + self._assert_display_topology() + + def generation_for_reserved_geometry( + self, + geometry: tuple[TargetWindow, float, float, tuple[int, int, int, int], int], + ) -> int: + """Return one receipt-time generation after delivery-time revalidation.""" + self._assert_reserved_geometry_current(geometry) + return geometry[4] + + def translate_reserved_geometry( + self, + geometry: tuple[TargetWindow, float, float, tuple[int, int, int, int], int], + x: float, + y: float, + ) -> tuple[float, float, int]: + """Translate against receipt-time geometry after exact revalidation.""" + self._assert_reserved_geometry_current(geometry) + window, scale_x, scale_y, content_rect, generation = geometry + return ( + (x - window.bounds[0]) * scale_x + content_rect[0], + (y - window.bounds[1]) * scale_y + content_rect[1], + generation, + ) + def generation_for_action(self) -> int: """Bind a non-pointer action to the exact published frame epoch.""" return self._geometry_for_action()[4] diff --git a/tests/test_capture_terminal.py b/tests/test_capture_terminal.py index 8c0d9ca..266eb6e 100644 --- a/tests/test_capture_terminal.py +++ b/tests/test_capture_terminal.py @@ -71,6 +71,8 @@ def _v2_capture_directory( *, frame_ordinals: tuple[int, ...] = (1, 3), action_ordinal: int | None = 2, + before_binding_ordinal: int | None = None, + after_binding_ordinal: int | None = None, wrong_png_digest: bool = False, video: bool = False, ) -> Path: @@ -156,8 +158,11 @@ def _v2_capture_directory( session.add_all((screenshot, window)) frames[ordinal] = (screenshot, window) if action_ordinal is not None: - before_ordinal = frame_ordinals[0] + before_ordinal = before_binding_ordinal or frame_ordinals[0] before, window = frames[before_ordinal] + after_ordinal = after_binding_ordinal or frame_ordinals[-1] + after, after_window = frames[after_ordinal] + has_after = after_ordinal > action_ordinal session.add( ActionEvent( recording_id=recording.id, @@ -172,10 +177,19 @@ def _v2_capture_directory( screenshot=before, screenshot_timestamp=before.timestamp, screenshot_source_ordinal=before_ordinal, + after_screenshot_timestamp=after.timestamp if has_after else None, + after_screenshot_source_ordinal=after_ordinal if has_after else None, window_event=window, window_event_timestamp=window.timestamp, window_event_source_ordinal=before_ordinal, + after_window_event_timestamp=( + after_window.timestamp if has_after else None + ), + after_window_event_source_ordinal=( + after_ordinal if has_after else None + ), window_geometry_generation=1, + after_window_geometry_generation=1 if has_after else None, ) ) session.commit() @@ -203,6 +217,116 @@ def _v2_capture_directory( return capture_dir +def _desktop_capture_directory( + root: Path, + *, + frame_ordinals: tuple[int, ...] = (1, 3), + action_ordinal: int | None = 2, + before_binding_ordinal: int | None = None, + after_binding_ordinal: int | None = None, + wrong_png_digest: bool = False, + retain_png: bool = True, +) -> Path: + """Build one sealed virtual-desktop capture for contract regressions.""" + capture_dir = root / "capture" + capture_dir.mkdir(parents=True) + config = { + "capture_desktop": { + "schema_version": "openadapt.capture.display-topology/v1", + "coordinate_space": "virtual_desktop_pixels", + "origin": [0, 0], + "viewport": [80, 60], + "monitor_count": 1, + "monitors": [[0, 0, 80, 60]], + "topology_sha256": "a" * 64, + } + } + + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + try: + recording = Recording( + timestamp=10.0, + monitor_width=80, + monitor_height=60, + platform="linux", + task_description="sealed desktop capture", + double_click_interval_seconds=0.5, + double_click_distance_pixels=5.0, + config=config, + ) + session.add(recording) + session.flush() + frames: dict[int, Screenshot] = {} + for index, ordinal in enumerate(frame_ordinals): + timestamp = 11.0 + index + output = io.BytesIO() + Image.new("RGB", (80, 60), (20 + index, 40, 60)).save( + output, + format="PNG", + ) + png = output.getvalue() + screenshot = Screenshot( + recording_id=recording.id, + recording_timestamp=10.0, + timestamp=timestamp, + source_ordinal=ordinal, + png_data=png if retain_png else None, + png_sha256=( + ("f" * 64 if wrong_png_digest else hashlib.sha256(png).hexdigest()) + if retain_png + else None + ), + ) + session.add(screenshot) + frames[ordinal] = screenshot + if action_ordinal is not None: + before_ordinal = before_binding_ordinal or frame_ordinals[0] + before = frames[before_ordinal] + after_ordinal = after_binding_ordinal or frame_ordinals[-1] + after = frames[after_ordinal] + has_after = after_ordinal > action_ordinal + session.add( + ActionEvent( + recording_id=recording.id, + recording_timestamp=10.0, + timestamp=11.5, + source_ordinal=action_ordinal, + name="click", + mouse_x=20.0, + mouse_y=20.0, + mouse_button_name="left", + mouse_pressed=False, + screenshot=before, + screenshot_timestamp=before.timestamp, + screenshot_source_ordinal=before_ordinal, + after_screenshot_timestamp=after.timestamp if has_after else None, + after_screenshot_source_ordinal=after_ordinal if has_after else None, + ) + ) + session.commit() + finally: + session.close() + engine.dispose() + + seal_capture( + capture_dir, + session_id="desktop-session", + process_started_at=9.0, + capture_started_at=10.0, + capture_ended_at=14.0, + event_counts={ + "action": int(action_ordinal is not None), + "screen": len(frame_ordinals), + "window": 0, + "browser": 0, + "video": 0, + }, + last_source_ordinal=max((*frame_ordinals, action_ordinal or 0)) or None, + ) + return capture_dir + + def test_terminal_binds_canonical_manifest_bytes_including_newline(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) terminal = _seal(capture_dir) @@ -219,17 +343,19 @@ def test_terminal_binds_canonical_manifest_bytes_including_newline(tmp_path) -> assert verified_terminal == terminal assert [artifact.path for artifact in manifest.artifacts] == [ "artifact.bin", + "capture-state.json", "recording.db", ] -def test_mutable_control_state_is_not_part_of_the_immutable_inventory(tmp_path) -> None: +def test_control_state_is_part_of_the_immutable_inventory(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) _seal(capture_dir) (capture_dir / "capture-state.json").write_text('{"phase":"complete"}\n') - verify_capture_artifacts(capture_dir) + with pytest.raises(CaptureSealError, match="differs from its seal"): + verify_capture_artifacts(capture_dir) def test_terminal_rejects_artifact_tamper_and_uninventoried_files(tmp_path) -> None: @@ -246,6 +372,31 @@ def test_terminal_rejects_artifact_tamper_and_uninventoried_files(tmp_path) -> N verify_capture_artifacts(other) +def test_seal_does_not_return_complete_after_an_artifact_changes_during_inventory( + tmp_path, + monkeypatch, +) -> None: + capture_dir = _capture_directory(tmp_path) + original_hash = terminal_module._hash_relative_regular_file + mutated = False + + def mutate_previous_artifact_then_hash(root, relative_path): + nonlocal mutated + if relative_path == "recording.db" and not mutated: + mutated = True + (capture_dir / "artifact.bin").write_bytes(b"changed after inventory") + return original_hash(root, relative_path) + + monkeypatch.setattr( + terminal_module, + "_hash_relative_regular_file", + mutate_previous_artifact_then_hash, + ) + + with pytest.raises(CaptureSealError, match="differs from its seal"): + _seal(capture_dir) + + def test_manifest_rejects_symbolic_links(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) (capture_dir / "linked.bin").symlink_to(capture_dir / "artifact.bin") @@ -306,6 +457,30 @@ def test_verified_loader_uses_a_private_snapshot_without_migrating_source(tmp_pa assert after == before +def test_sealed_semantic_validation_copies_only_the_database( + tmp_path, + monkeypatch, +) -> None: + capture_dir = _capture_directory(tmp_path) + _seal(capture_dir) + copied: list[str] = [] + original_copy = terminal_module._copy_verified_regular_file + + def track_copy(source_root, relative_path, destination, expected): + copied.append(relative_path) + return original_copy(source_root, relative_path, destination, expected) + + monkeypatch.setattr( + terminal_module, + "_copy_verified_regular_file", + track_copy, + ) + + CaptureSession.validate_sealed(capture_dir) + + assert copied == ["recording.db"] + + def test_verified_loader_rejects_terminal_counts_that_differ_from_database(tmp_path) -> None: capture_dir = _capture_directory(tmp_path) seal_capture( @@ -389,7 +564,7 @@ def test_verified_loader_rejects_duplicate_source_ordinals(tmp_path) -> None: def test_verified_loader_requires_a_v2_after_frame_for_every_action(tmp_path) -> None: capture_dir = _v2_capture_directory(tmp_path, frame_ordinals=(1,)) - with pytest.raises(ValueError, match="no ordinal-later retained after frame"): + with pytest.raises(ValueError, match="incomplete before/after frame binding"): CaptureSession.load_verified(capture_dir) @@ -400,6 +575,32 @@ def test_verified_loader_accepts_a_complete_v2_source_journal(tmp_path) -> None: assert [frame.source_ordinal for frame in capture.frames()] == [1, 3] +def test_verified_loader_rejects_a_skipped_nearest_before_frame(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 2, 4), + action_ordinal=3, + before_binding_ordinal=1, + after_binding_ordinal=4, + ) + + with pytest.raises(ValueError, match="nearest retained before frame"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_skipped_first_after_frame(tmp_path) -> None: + capture_dir = _v2_capture_directory( + tmp_path, + frame_ordinals=(1, 3, 4), + action_ordinal=2, + before_binding_ordinal=1, + after_binding_ordinal=4, + ) + + with pytest.raises(ValueError, match="first retained after frame"): + CaptureSession.load_verified(capture_dir) + + def test_verified_loader_rejects_a_gap_in_the_v2_source_journal(tmp_path) -> None: capture_dir = _v2_capture_directory( tmp_path, @@ -411,6 +612,74 @@ def test_verified_loader_rejects_a_gap_in_the_v2_source_journal(tmp_path) -> Non CaptureSession.load_verified(capture_dir) +def test_verified_loader_requires_a_desktop_after_frame_for_every_action( + tmp_path, +) -> None: + capture_dir = _desktop_capture_directory(tmp_path, frame_ordinals=(1,)) + + with pytest.raises(ValueError, match="incomplete before/after frame binding"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_skipped_desktop_nearest_before_frame( + tmp_path, +) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(1, 2, 4), + action_ordinal=3, + before_binding_ordinal=1, + after_binding_ordinal=4, + ) + + with pytest.raises(ValueError, match="nearest retained before frame"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_skipped_desktop_first_after_frame( + tmp_path, +) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(1, 3, 4), + action_ordinal=2, + before_binding_ordinal=1, + after_binding_ordinal=4, + ) + + with pytest.raises(ValueError, match="first retained after frame"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_recomputes_desktop_png_digest(tmp_path) -> None: + capture_dir = _desktop_capture_directory(tmp_path, wrong_png_digest=True) + + with pytest.raises(ValueError, match="PNG digest differs"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_desktop_capture_without_a_retained_frame( + tmp_path, +) -> None: + capture_dir = _desktop_capture_directory( + tmp_path, + frame_ordinals=(), + action_ordinal=None, + ) + + with pytest.raises(ValueError, match="no retained frames"): + CaptureSession.load_verified(capture_dir) + + +def test_verified_loader_rejects_a_desktop_frame_without_a_pixel_carrier( + tmp_path, +) -> None: + capture_dir = _desktop_capture_directory(tmp_path, retain_png=False) + + with pytest.raises(ValueError, match="no retained PNG or exact MP4 carrier"): + CaptureSession.load_verified(capture_dir) + + def test_verified_loader_recomputes_retained_v2_png_digest(tmp_path) -> None: capture_dir = _v2_capture_directory(tmp_path, wrong_png_digest=True) @@ -464,6 +733,23 @@ def test_verified_loader_accepts_v2_mp4_bindings_joined_to_database_frames( assert capture.video_path is not None +def test_verified_loader_rejects_an_extra_v2_mp4_capture_binding( + tmp_path, + monkeypatch, +) -> None: + capture_dir = _v2_capture_directory(tmp_path, video=True) + timing = ( + None, + [(0, 0.0), (1, 1.0), (2, 2.0)], + [(0, 11.0), (1, 12.0), (2, 13.0)], + [(0, 1), (1, 3)], + ) + monkeypatch.setattr("openadapt_capture.video._read_timing_metadata", lambda _path: timing) + + with pytest.raises(ValueError, match="capture-time bindings differ"): + CaptureSession.load_verified(capture_dir) + + def test_verified_loader_rejects_multiple_recognized_video_artifacts(tmp_path) -> None: capture_dir = _v2_capture_directory(tmp_path, video=True) # Add the second recognized name before replacing the immutable seal. diff --git a/tests/test_control.py b/tests/test_control.py index 3d0911e..aa8ed84 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import io import json import multiprocessing import os @@ -18,6 +20,7 @@ import psutil import pytest +from PIL import Image from openadapt_capture import control from openadapt_capture import recorder as recorder_module @@ -31,6 +34,7 @@ stop_recording, ) from openadapt_capture.db import create_db, crud +from openadapt_capture.terminal import verify_capture_artifacts def _terminal_payload(capture_dir: Path, session_id: str) -> dict: @@ -164,6 +168,10 @@ def test_subprocess_ready_status_stop_and_complete(tmp_path: Path) -> None: assert terminal["integrity_verified"] is True assert "token" not in terminal assert "capture_dir" not in terminal + _, manifest = verify_capture_artifacts(capture_dir) + assert "capture-state.json" in { + artifact.path for artifact in manifest.artifacts + } assert discover_recorders(runtime_dir) == [] finally: if child.poll() is None: @@ -506,6 +514,45 @@ def test_pid_reuse_descriptor_is_marked_crashed_and_removed(tmp_path: Path) -> N assert recovered["complete"] is False +def test_staged_complete_state_without_a_seal_is_marked_crashed( + tmp_path: Path, +) -> None: + capture_dir = tmp_path / "capture" + session_id = str(uuid.uuid4()) + process_started_at = psutil.Process().create_time() - 100.0 + staged = _terminal_payload(capture_dir, session_id) + staged.update( + { + "process_started_at": process_started_at, + "phase": "complete", + "complete": True, + "integrity_verified": True, + "finalized_at": time.time(), + } + ) + control.write_terminal_state(capture_dir, staged) + descriptor = control._ControlDescriptor( + session_id=session_id, + pid=os.getpid(), + process_started_at=process_started_at, + capture_dir=str(capture_dir), + host="127.0.0.1", + port=65534, + created_at=time.time(), + path=tmp_path / "unused.json", + token="x" * 64, + ) + + control._mark_crashed_if_bound(descriptor) + + recovered = json.loads( + (capture_dir / control.TERMINAL_STATE_FILENAME).read_text(encoding="utf-8") + ) + assert recovered["phase"] == "crashed" + assert recovered["complete"] is False + assert recovered["integrity_verified"] is False + + def test_exited_but_inspectable_process_is_not_live( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -774,6 +821,60 @@ def test_integrity_verification_rejects_missing_committed_events(tmp_path: Path) recorder._verify_completed_capture() +def test_completion_revalidates_the_exact_sealed_database_snapshot( + tmp_path: Path, +) -> None: + capture_dir = tmp_path / "capture" + _create_minimal_recording(capture_dir) + output = io.BytesIO() + Image.new("RGB", (2, 2), "black").save(output, format="PNG") + png = output.getvalue() + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + try: + recording = session.query(crud.Recording).one() + crud.insert_screenshot( + session, + recording, + recording.timestamp + 1, + { + "source_ordinal": 1, + "png_data": png, + "png_sha256": hashlib.sha256(png).hexdigest(), + }, + ) + finally: + session.close() + engine.dispose() + + recorder = recorder_module.Recorder( + str(capture_dir), + capture_video=False, + capture_images=True, + ) + recorder._num_screen_events.value = 1 + recorder._last_source_ordinal = 1 + recorder._verify_completed_capture() + + database = create_db(str(capture_dir / "recording.db"))[0] + try: + with database.begin() as connection: + connection.exec_driver_sql( + "UPDATE screenshot SET png_sha256 = ?", + ("f" * 64,), + ) + finally: + database.dispose() + + recorder._stage_completed_control_state() + with pytest.raises(ValueError, match="PNG digest differs"): + recorder._seal_completed_capture() + + status = recorder._control_payload() + assert status["complete"] is False + assert status["integrity_verified"] is False + + def test_integrity_verification_rejects_malformed_browser_event(tmp_path: Path) -> None: capture_dir = tmp_path / "capture" _create_minimal_recording(capture_dir, browser_messages=["not-an-object"]) diff --git a/tests/test_desktop_capture.py b/tests/test_desktop_capture.py index 2783c5c..c8b51d8 100644 --- a/tests/test_desktop_capture.py +++ b/tests/test_desktop_capture.py @@ -3,14 +3,26 @@ from __future__ import annotations import json +import queue +import threading +import time +from types import SimpleNamespace import pytest +from PIL import Image +import openadapt_capture.recorder as recorder_module from openadapt_capture import utils from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud from openadapt_capture.desktop_capture import DesktopCaptureError, DesktopCaptureScope -from openadapt_capture.recorder import create_recording, trigger_action_event +from openadapt_capture.recorder import ( + NativeInputFrameBoundary, + OrderedEventJournal, + create_recording, + read_screen_events, + trigger_action_event, +) def _two_monitor_scope() -> DesktopCaptureScope: @@ -151,3 +163,109 @@ def test_recording_rejects_ambiguous_coordinate_scopes(tmp_path) -> None: window_capture_info={"coordinate_space": "window_pixels"}, desktop_capture_info={"coordinate_space": "virtual_desktop_pixels"}, ) + + +def test_desktop_screen_reader_discards_a_frame_crossed_by_native_input( + monkeypatch, +) -> None: + monkeypatch.setattr(recorder_module.config, "SCREEN_CAPTURE_FPS", 0) + boundary = NativeInputFrameBoundary() + clean_results = iter((False, True)) + completed_tokens: list[object] = [] + terminate = threading.Event() + + class FakeObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return next(clean_results) + + def complete_frame_capture(self, token: object) -> None: + completed_tokens.append(token) + + boundary.attach(FakeObserver()) + capture_calls = 0 + + def take_screenshot() -> Image.Image: + nonlocal capture_calls + capture_calls += 1 + if capture_calls == 2: + terminate.set() + return Image.new("RGB", (4480, 1440), "black") + + monkeypatch.setattr(recorder_module.utils, "take_screenshot", take_screenshot) + journal = OrderedEventJournal() + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + desktop_scope=_two_monitor_scope(), + input_frame_boundary=boundary, + ) + + assert capture_calls == 2 + assert len(completed_tokens) == 2 + frame = journal.get_nowait() + assert frame.type == "screen" + assert frame.source_ordinal == 1 + with pytest.raises(queue.Empty): + journal.get_nowait() + + +def test_desktop_terminal_frame_seals_input_before_journal_commit( + monkeypatch, +) -> None: + terminate = threading.Event() + terminate.set() + input_finished = threading.Event() + input_finished.set() + terminal_finished = threading.Event() + boundary = NativeInputFrameBoundary() + sealed = False + completed = False + + class FakeObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return True + + def seal_frame_capture(self, _token: object) -> None: + nonlocal sealed + sealed = True + + def complete_frame_capture(self, _token: object) -> None: + nonlocal completed + completed = True + + boundary.attach(FakeObserver()) + monkeypatch.setattr( + recorder_module.utils, + "take_screenshot", + lambda: Image.new("RGB", (4480, 1440), "black"), + ) + journal = OrderedEventJournal() + original_put = journal.put + + def checked_put(*args, **kwargs): + assert sealed + return original_put(*args, **kwargs) + + monkeypatch.setattr(journal, "put", checked_put) + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + desktop_scope=_two_monitor_scope(), + input_finished=input_finished, + input_frame_boundary=boundary, + terminal_frame_finished=terminal_finished, + ) + + assert terminal_finished.is_set() + assert completed + assert journal.get_nowait().type == "screen" diff --git a/tests/test_frame_binding.py b/tests/test_frame_binding.py index 0c814a3..17b6cd2 100644 --- a/tests/test_frame_binding.py +++ b/tests/test_frame_binding.py @@ -355,29 +355,60 @@ def _click_pair(down_ts=10.0, up_ts=10.05, down_bound=9.9, up_bound=10.0): return down, up -def test_click_merge_keeps_the_up_childs_binding(): +def test_click_merge_keeps_the_down_childs_before_binding(): down, up = _click_pair() (merged,) = merge_consecutive_mouse_click_events([down, up]) assert isinstance(merged, MouseClickEvent) - assert merged.screenshot_timestamp == 10.0 + assert merged.screenshot_timestamp == 9.9 -def test_drag_merge_keeps_the_final_bindings(): +def test_click_merge_keeps_the_terminal_childs_source_and_after_bindings(): + down = MouseDownEvent( + timestamp=10.0, + source_ordinal=2, + x=1.0, + y=2.0, + button="left", + screenshot_timestamp=9.9, + screenshot_source_ordinal=1, + after_screenshot_timestamp=10.0, + after_screenshot_source_ordinal=3, + ) + up = MouseUpEvent( + timestamp=10.05, + source_ordinal=4, + x=1.0, + y=2.0, + button="left", + screenshot_timestamp=10.0, + screenshot_source_ordinal=3, + after_screenshot_timestamp=10.1, + after_screenshot_source_ordinal=5, + ) + + (merged,) = merge_consecutive_mouse_click_events([down, up]) + + assert merged.source_ordinal == 4 + assert merged.screenshot_source_ordinal == 1 + assert merged.after_screenshot_source_ordinal == 5 + + +def test_drag_merge_keeps_the_initial_before_binding(): down = MouseDownEvent(timestamp=1.0, x=1.0, y=2.0, button="left", screenshot_timestamp=0.9) move = MouseMoveEvent(timestamp=1.5, x=30.0, y=30.0, screenshot_timestamp=1.6) up = MouseUpEvent(timestamp=2.0, x=30.0, y=30.0, button="left", screenshot_timestamp=2.0) (drag,) = detect_drag_events([down, move, up]) assert isinstance(drag, MouseDragEvent) - assert drag.screenshot_timestamp == 2.0 + assert drag.screenshot_timestamp == 0.9 -def test_type_merge_keeps_the_last_key_binding(): +def test_type_merge_keeps_the_first_key_before_binding(): first = KeyDownEvent(timestamp=20.0, key_char="a", screenshot_timestamp=19.5) second = KeyDownEvent(timestamp=21.0, key_char="b", screenshot_timestamp=20.75) (merged,) = merge_consecutive_keyboard_events([first, second]) assert isinstance(merged, KeyTypeEvent) assert merged.text == "ab" - assert merged.screenshot_timestamp == 20.75 + assert merged.screenshot_timestamp == 19.5 def test_merge_leaves_legacy_children_unbound(): @@ -421,6 +452,25 @@ def test_action_screenshot_uses_exact_binding(): assert image.getpixel((0, 0)) == (255, 0, 0) +def test_action_after_screenshot_uses_exact_binding(): + stub = _StubCapture() + event = MouseUpEvent( + timestamp=10.05, + x=1.0, + y=2.0, + button="left", + after_screenshot_timestamp=10.1, + after_screenshot_source_ordinal=5, + ) + + image = Action(event=event, _capture=stub).after_screenshot + + assert image is not None + assert stub.exact_calls == [(10.1, 5)] + assert stub.lenient_calls == [] + assert image.getpixel((0, 0)) == (255, 0, 0) + + def test_capture_session_joins_database_action_to_exact_mp4_source_frame( tmp_path, monkeypatch, diff --git a/tests/test_highlevel.py b/tests/test_highlevel.py index b884fb6..93a7e58 100644 --- a/tests/test_highlevel.py +++ b/tests/test_highlevel.py @@ -19,6 +19,7 @@ from openadapt_capture.db import create_db, crud from openadapt_capture.platform import DisplayMetricsUnavailable from openadapt_capture.recorder import Recorder +from openadapt_capture.terminal import CaptureSealError # Sessions/engines created by _create_test_recording, released by the # temp_capture_dir teardown BEFORE the TemporaryDirectory is removed. Both @@ -201,6 +202,9 @@ def test_recorder_capture_waits_for_seal_and_uses_verified_loader( rec._record_thread = SimpleNamespace(is_alive=lambda: True) rec._terminate_processing.set() assert rec.capture is None + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + (capture_dir / "recording.db").touch() loaded = object() monkeypatch.setattr( @@ -212,6 +216,16 @@ def test_recorder_capture_waits_for_seal_and_uses_verified_loader( assert rec.capture is loaded + def test_recorder_capture_does_not_hide_missing_seal_metadata(self, tmp_path): + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + (capture_dir / "recording.db").touch() + rec = Recorder(str(capture_dir)) + rec._finalized_event.set() + + with pytest.raises(CaptureSealError, match="capture artifact is missing"): + _ = rec.capture + def test_recorder_screen_count_property(self): """Test Recorder has screen_count property.""" rec = Recorder("/tmp/test_never_created") diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index c94a1d6..b820dc4 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -416,6 +416,225 @@ def consume(event) -> None: assert [event.timestamp for event in public_events] == [10.25, 10.5] +def test_stop_fails_a_native_receipt_that_never_reached_delivery() -> None: + journal = recorder_module.OrderedEventJournal() + + def consume(_event) -> None: + return + + setattr(consume, "_openadapt_input_receipt", journal.reserve) + observer = _ReadyObserver(consume) + observer.start() + receipt = observer._reserve_receipt(1.0) + + with pytest.raises(InputObserverError, match="stopped before reserved native input"): + observer.stop() + + assert receipt is not None + assert receipt.finished + with pytest.raises(recorder_module.EventJournalReservationError): + journal.get_nowait() + + +def test_terminal_frame_seal_keeps_prior_receipts_and_drops_later_input() -> None: + journal = recorder_module.OrderedEventJournal() + + def consume(_event) -> None: + pytest.fail("reserved input must use the receipt consumer") + + def deliver(event, receipt) -> None: + receipt.complete( + recorder_module.Event(event.timestamp, "action", {"key": event.key_char}) + ) + + setattr(consume, "_openadapt_input_receipt", journal.reserve) + setattr(consume, "_openadapt_input_delivery", deliver) + observer = _ReadyObserver(consume) + observer.start() + + accepted = ObservedKey(pressed=True, key_char="a", timestamp=1.0) + accepted_receipt = observer._reserve_receipt(accepted.timestamp) + observer.seal_frame_capture(None) + observer._emit_received(accepted, accepted_receipt) + + outside = ObservedKey(pressed=True, key_char="b", timestamp=2.0) + outside_receipt = observer._reserve_receipt(outside.timestamp) + observer._emit_received(outside, outside_receipt) + + deadline = time.monotonic() + 1 + while not bool(getattr(accepted_receipt, "finished", False)): + if time.monotonic() >= deadline: + pytest.fail("the pre-seal receipt did not finish") + time.sleep(0.001) + observer.stop() + + action = journal.get_nowait() + assert (action.data["key"], action.source_ordinal) == ("a", 1) + with pytest.raises(queue.Empty): + journal.get_nowait() + + +def test_generic_frame_cut_rejects_dirty_pixels_and_orders_post_cut_input() -> None: + journal = recorder_module.OrderedEventJournal() + + def consume(_event) -> None: + pytest.fail("reserved input must use the receipt consumer") + + def deliver(event, receipt) -> None: + receipt.complete( + recorder_module.Event(event.timestamp, "action", {"key": event.key_char}) + ) + + setattr(consume, "_openadapt_input_receipt", journal.reserve) + setattr(consume, "_openadapt_input_delivery", deliver) + observer = _ReadyObserver(consume) + observer.start() + + dirty_cut = observer.begin_frame_capture() + dirty_event = ObservedKey(pressed=True, key_char="a", timestamp=1.0) + dirty_receipt = observer._reserve_receipt(dirty_event.timestamp) + assert not observer.finish_frame_capture(dirty_cut) + observer.complete_frame_capture(dirty_cut) + observer._emit_received(dirty_event, dirty_receipt) + + clean_cut = observer.begin_frame_capture() + assert observer.finish_frame_capture(clean_cut) + reserve_started = threading.Event() + post_cut: dict[str, object] = {} + + def reserve_post_cut_input() -> None: + reserve_started.set() + post_cut["receipt"] = observer._reserve_receipt(3.0) + + reserve_thread = threading.Thread(target=reserve_post_cut_input) + reserve_thread.start() + assert reserve_started.wait(timeout=1) + time.sleep(0.01) + assert "receipt" not in post_cut + journal.put(recorder_module.Event(2.0, "screen", {})) + observer.complete_frame_capture(clean_cut) + reserve_thread.join(timeout=1) + assert not reserve_thread.is_alive() + + post_cut_event = ObservedKey(pressed=True, key_char="b", timestamp=3.0) + observer._emit_received(post_cut_event, post_cut["receipt"]) + deadline = time.monotonic() + 1 + while not bool(getattr(post_cut["receipt"], "finished", False)): + if time.monotonic() >= deadline: + pytest.fail("the post-cut receipt did not finish") + time.sleep(0.001) + observer.stop() + + assert [journal.get_nowait().type for _ in range(3)] == [ + "action", + "screen", + "action", + ] + + +def test_consumer_failure_fails_later_queued_receipts() -> None: + delivery_entered = threading.Event() + release_delivery = threading.Event() + receipts = [] + + class Receipt: + def __init__(self) -> None: + self.finished = False + + def fail(self, _error) -> None: + self.finished = True + + def consume(_event) -> None: + raise AssertionError("reserved input must use the receipt consumer") + + def reserve(_timestamp): + receipt = Receipt() + receipts.append(receipt) + return receipt + + def deliver(_event, _receipt) -> None: + delivery_entered.set() + assert release_delivery.wait(timeout=1) + raise RuntimeError("consumer failed") + + setattr(consume, "_openadapt_input_receipt", reserve) + setattr(consume, "_openadapt_input_delivery", deliver) + observer = _ReadyObserver(consume) + observer.start() + first = ObservedKey(pressed=True, key_char="a", timestamp=1.0) + observer._emit(first, receipt=observer._reserve_receipt(first.timestamp)) + assert delivery_entered.wait(timeout=1) + second = ObservedKey(pressed=False, key_char="a", timestamp=2.0) + observer._emit(second, receipt=observer._reserve_receipt(second.timestamp)) + release_delivery.set() + + deadline = time.monotonic() + 1 + while not all(receipt.finished for receipt in receipts): + if time.monotonic() >= deadline: + pytest.fail("queued native receipts were not failed") + time.sleep(0.001) + + with pytest.raises(InputObserverError, match="consumer failed"): + observer.stop() + + +def test_delivery_start_hook_failure_fails_an_already_queued_receipt() -> None: + journal = recorder_module.OrderedEventJournal() + hook_entered = threading.Event() + release_hook = threading.Event() + + class Callback: + def __call__(self, _event) -> None: + pytest.fail("reserved input must use the receipt consumer") + + def _openadapt_input_receipt(self, timestamp): + return journal.reserve(timestamp) + + def _openadapt_input_delivery(self, _event, _receipt) -> None: + pytest.fail("the failed delivery hook must prevent event delivery") + + def _openadapt_delivery_thread_start(self) -> None: + hook_entered.set() + assert release_hook.wait(timeout=1) + raise RuntimeError("delivery start hook failed") + + class SetupReceiptObserver(_ReadyObserver): + def _setup(self) -> None: + event = ObservedKey(pressed=True, key_char="a", timestamp=1.0) + self.receipt = self._reserve_receipt(event.timestamp) + self._emit_received(event, self.receipt) + + observer = SetupReceiptObserver(Callback()) + start_errors: list[BaseException] = [] + + def start_observer() -> None: + try: + observer.start() + except BaseException as exc: + start_errors.append(exc) + + start_thread = threading.Thread(target=start_observer) + start_thread.start() + assert hook_entered.wait(timeout=1) + assert start_thread.is_alive() + release_hook.set() + start_thread.join(timeout=1) + assert not start_thread.is_alive() + assert len(start_errors) == 1 + assert isinstance(start_errors[0], InputObserverError) + assert "delivery start hook failed" in str(start_errors[0]) + + deadline = time.monotonic() + 1 + while not observer.receipt.finished: + if time.monotonic() >= deadline: + pytest.fail("delivery start failure did not fail the queued receipt") + time.sleep(0.001) + + assert observer._delivery_queue.empty() + with pytest.raises(recorder_module.EventJournalReservationError): + journal.get_nowait() + + def test_stop_sequence_callback_can_stop_listener_from_delivery_thread( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_input_observer_darwin.py b/tests/test_input_observer_darwin.py index 7643ce4..3c3e630 100644 --- a/tests/test_input_observer_darwin.py +++ b/tests/test_input_observer_darwin.py @@ -2,6 +2,9 @@ from __future__ import annotations +import os +import sys +import threading import time from types import SimpleNamespace @@ -66,8 +69,10 @@ class FakeQuartz: kCGEventFlagMaskSecondaryFn = 1 << 5 kCGHIDEventTap = 300 + kCGSessionEventTap = 303 kCGHeadInsertEventTap = 301 kCGEventTapOptionListenOnly = 302 + kCGEventTapOptionDefault = 304 kCFRunLoopDefaultMode = "default" def __init__( @@ -115,9 +120,9 @@ def CGEventTapCreate( callback, refcon, ): - assert tap_location == self.kCGHIDEventTap + assert tap_location == self.kCGSessionEventTap assert placement == self.kCGHeadInsertEventTap - assert options == self.kCGEventTapOptionListenOnly + assert options == self.kCGEventTapOptionDefault assert refcon is None self.event_mask = event_mask self.callback = callback @@ -259,13 +264,26 @@ def test_accessibility_permission_is_fallback_on_older_macos() -> None: assert quartz.callback is None +def test_active_event_barrier_requires_accessibility_permission() -> None: + quartz = FakeQuartz() + observer = make_observer( + quartz, + lambda _event: None, + application_services=FakeApplicationServices(trusted=False), + ) + + with pytest.raises(InputObserverPermissionError, match="ordered input barrier"): + observer.start() + assert quartz.callback is None + + def test_event_tap_creation_failure_is_explicit() -> None: quartz = FakeQuartz(create_tap=False) observer = make_observer(quartz, lambda _event: None) with pytest.raises( InputObserverUnavailableError, - match="listen-only Quartz event tap", + match="active Quartz event barrier", ): observer.start() @@ -328,6 +346,267 @@ def test_mouse_move_button_and_scroll_normalization() -> None: ] +def test_injected_transition_invalidates_an_in_flight_frame() -> None: + quartz = FakeQuartz() + observer = make_observer(quartz, lambda _event: None) + observer.start() + cut = observer.begin_frame_capture() + + observer._handle_event( + quartz.kCGEventKeyDown, + FakeEvent( + fields={ + quartz.kCGEventSourceUnixProcessID: 4242, + quartz.kCGKeyboardEventKeycode: 0, + }, + text="a", + ), + timestamp=1.0, + ) + + assert not observer.finish_frame_capture(cut) + observer.complete_frame_capture(cut) + observer.stop() + + +def test_native_callback_active_before_frame_invalidates_the_cut( + monkeypatch: pytest.MonkeyPatch, +) -> None: + quartz = FakeQuartz() + observer = make_observer(quartz, lambda _event: None) + callback_reserved = threading.Event() + release_callback = threading.Event() + original_emit = observer._emit + + def hold_after_reservation(event, *, receipt=None) -> None: + callback_reserved.set() + assert release_callback.wait(timeout=2) + original_emit(event, receipt=receipt) + + monkeypatch.setattr(observer, "_emit", hold_after_reservation) + observer.start() + callback_thread = threading.Thread( + target=observer._event_callback, + args=( + None, + quartz.kCGEventKeyDown, + FakeEvent( + fields={ + quartz.kCGEventSourceUnixProcessID: 0, + quartz.kCGKeyboardEventKeycode: 0, + }, + text="a", + ), + None, + ), + ) + callback_thread.start() + assert callback_reserved.wait(timeout=2) + + cut = observer.begin_frame_capture() + release_callback.set() + callback_thread.join(timeout=2) + assert not callback_thread.is_alive() + assert not observer.finish_frame_capture(cut) + observer.complete_frame_capture(cut) + observer.stop() + + +def test_frame_waits_for_quartz_to_accept_the_returned_event() -> None: + quartz = FakeQuartz() + observer = make_observer(quartz, lambda _event: None) + + observer._event_callback( + None, + quartz.kCGEventKeyDown, + FakeEvent( + fields={ + quartz.kCGEventSourceUnixProcessID: 0, + quartz.kCGKeyboardEventKeycode: 0, + }, + text="a", + ), + None, + ) + crossed_cut = observer.begin_frame_capture() + + observer._complete_delivery_barriers() + assert not observer.finish_frame_capture(crossed_cut) + observer.complete_frame_capture(crossed_cut) + + clean_cut = observer.begin_frame_capture() + assert observer.finish_frame_capture(clean_cut) + observer.complete_frame_capture(clean_cut) + + +@pytest.mark.slow +@pytest.mark.skipif(sys.platform != "darwin", reason="requires macOS Quartz") +@pytest.mark.skipif( + os.getenv("OPENADAPT_DARWIN_EVENT_TAP_SMOKE") != "1", + reason="set OPENADAPT_DARWIN_EVENT_TAP_SMOKE=1 on an interactive macOS runner", +) +def test_active_tap_holds_annotated_delivery_until_frame_commit() -> None: + import ApplicationServices + import Quartz + + downstream_ready = threading.Event() + downstream_received = threading.Event() + downstream_stop = threading.Event() + barrier_entered = threading.Event() + marker = 0x0A0A_DA7A + downstream_state: dict[str, object] = {} + + def downstream_callback(_proxy, _event_type, event, _refcon): + value = Quartz.CGEventGetIntegerValueField( + event, + Quartz.kCGEventSourceUserData, + ) + if value == marker: + downstream_received.set() + return event + + def downstream_main() -> None: + callback_ref = downstream_callback + event_mask = Quartz.CGEventMaskBit(Quartz.kCGEventMouseMoved) + tap = Quartz.CGEventTapCreate( + Quartz.kCGAnnotatedSessionEventTap, + Quartz.kCGTailAppendEventTap, + Quartz.kCGEventTapOptionListenOnly, + event_mask, + callback_ref, + None, + ) + assert tap is not None + source = Quartz.CFMachPortCreateRunLoopSource(None, tap, 0) + assert source is not None + loop = Quartz.CFRunLoopGetCurrent() + downstream_state.update(tap=tap, source=source, loop=loop, callback=callback_ref) + Quartz.CFRunLoopAddSource(loop, source, Quartz.kCFRunLoopDefaultMode) + Quartz.CGEventTapEnable(tap, True) + downstream_ready.set() + while not downstream_stop.is_set(): + Quartz.CFRunLoopRunInMode( + Quartz.kCFRunLoopDefaultMode, + 0.05, + False, + ) + Quartz.CGEventTapEnable(tap, False) + Quartz.CFRunLoopRemoveSource(loop, source, Quartz.kCFRunLoopDefaultMode) + Quartz.CFRunLoopSourceInvalidate(source) + Quartz.CFMachPortInvalidate(tap) + + class BarrierProbeObserver(DarwinInputObserver): + def _mark_native_activity(self) -> None: + barrier_entered.set() + super()._mark_native_activity() + + downstream_thread = threading.Thread(target=downstream_main, daemon=True) + downstream_thread.start() + assert downstream_ready.wait(timeout=2) + observer = BarrierProbeObserver( + lambda _event: None, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + startup_timeout=2, + shutdown_timeout=2, + _quartz=Quartz, + _application_services=ApplicationServices, + ) + observer.start() + cut = observer.begin_frame_capture() + assert observer.finish_frame_capture(cut) + + current = Quartz.CGEventCreate(None) + location = Quartz.CGEventGetLocation(current) + event = Quartz.CGEventCreateMouseEvent( + None, + Quartz.kCGEventMouseMoved, + location, + Quartz.kCGMouseButtonLeft, + ) + Quartz.CGEventSetIntegerValueField( + event, + Quartz.kCGEventSourceUserData, + marker, + ) + Quartz.CGEventPost(Quartz.kCGSessionEventTap, event) + + try: + assert barrier_entered.wait(timeout=2) + assert not downstream_received.wait(timeout=0.1) + observer.complete_frame_capture(cut) + assert downstream_received.wait(timeout=2) + finally: + observer.complete_frame_capture(cut) + observer.stop() + downstream_stop.set() + loop = downstream_state.get("loop") + if loop is not None: + Quartz.CFRunLoopStop(loop) + downstream_thread.join(timeout=2) + assert not downstream_thread.is_alive() + + +def test_event_handler_reserves_receipt_before_mouse_normalization() -> None: + quartz = FakeQuartz() + location_entered = threading.Event() + release_location = threading.Event() + reserved_timestamps = [] + events = [] + + def blocked_location(event): + location_entered.set() + assert release_location.wait(timeout=1) + return event.location + + quartz.CGEventGetLocation = blocked_location + + class Receipt: + finished = False + + def fail(self, _error) -> None: + self.finished = True + + def consume(_event) -> None: + raise AssertionError("reserved input must use the receipt consumer") + + def reserve(timestamp): + reserved_timestamps.append(timestamp) + return Receipt() + + def deliver(event, receipt) -> None: + events.append(event) + receipt.finished = True + + setattr(consume, "_openadapt_input_receipt", reserve) + setattr(consume, "_openadapt_input_delivery", deliver) + observer = make_observer(quartz, consume) + observer.start() + handler = threading.Thread( + target=observer._handle_event, + args=( + quartz.kCGEventMouseMoved, + FakeEvent(fields={quartz.kCGEventSourceUnixProcessID: 0}), + ), + kwargs={"timestamp": 111.25}, + ) + handler.start() + + assert location_entered.wait(timeout=1) + assert reserved_timestamps == [111.25] + assert events == [] + + release_location.set() + handler.join(timeout=1) + deadline = time.monotonic() + 1 + while not events: + if time.monotonic() >= deadline: + pytest.fail("reserved macOS event was not delivered") + time.sleep(0.001) + observer.stop() + + def test_move_filter_does_not_disable_buttons() -> None: quartz = FakeQuartz() events = [] diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index c49f366..8eef2ad 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -4,6 +4,7 @@ import ctypes import sys +import threading import time import pytest @@ -263,6 +264,80 @@ def intercept(payload: bytes, *, id_base: int) -> None: assert len(freed) == 3 +def test_frame_cut_rejects_batched_input_and_blocks_post_marker_records() -> None: + observer = make_observer() + observer._setup_complete = True + observer._accepting_events = True + observer._control_display = object() + observer._root = 1 + observer._control_id_base = 0x400000 + callback_threads: list[threading.Thread] = [] + include_device_before_marker = True + post_marker_processed = threading.Event() + + class FakeXtst: + def XRecordFreeData(self, _pointer) -> None: + return + + class FakeX11: + def XQueryPointer(self, *_args) -> int: + def deliver_batch() -> None: + if include_device_before_marker: + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=100, + ), + ) + marker = bytearray(32) + marker[0] = 1 + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=bytes(marker), + id_base=observer._control_id_base, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._MOTION_NOTIFY, + detail=0, + event_time=101, + root_x=12, + root_y=34, + ), + ) + post_marker_processed.set() + + thread = threading.Thread(target=deliver_batch) + callback_threads.append(thread) + thread.start() + return 1 + + observer._xtst = FakeXtst() + observer._x11 = FakeX11() + + dirty = observer.begin_frame_capture() + assert not observer.finish_frame_capture(dirty) + assert not post_marker_processed.is_set() + observer.complete_frame_capture(dirty) + callback_threads[-1].join(timeout=1) + assert post_marker_processed.is_set() + + include_device_before_marker = False + post_marker_processed.clear() + clean = observer.begin_frame_capture() + assert observer.finish_frame_capture(clean) + assert not post_marker_processed.is_set() + observer.complete_frame_capture(clean) + callback_threads[-1].join(timeout=1) + assert post_marker_processed.is_set() + + def test_xkb_lookup_uses_state_from_delivered_event() -> None: observer = make_observer() x11 = FakeX11(keysym=0x20AC, keysym_name="EuroSign") @@ -531,6 +606,57 @@ def test_normative_device_then_duplicate_deliveries_then_next_device_order( assert x11.lookup_states == [0x0001, 0x0001] +def test_device_stream_reserves_receipt_before_delivered_event_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reserved_timestamps = [] + + class Receipt: + finished = False + + def fail(self, _error) -> None: + self.finished = True + + def consume(_event) -> None: + return + + def reserve(timestamp): + reserved_timestamps.append(timestamp) + return Receipt() + + setattr(consume, "_openadapt_input_receipt", reserve) + observer = LinuxXInputObserver( + consume, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + environ={"DISPLAY": ":0", "XDG_SESSION_TYPE": "x11"}, + ) + monkeypatch.setattr(linux_module.time, "time", lambda: 100.0) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 200.0) + + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 900, + 0, + 0, + 0, + 50, + 60, + 0, + 0, + 0x0001, + ) + ) + + assert reserved_timestamps == [100.0] + assert observer._pending is not None + assert observer._pending.receipt is not None + + def test_unmatched_key_keeps_physical_identity_and_suppresses_later_text( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_input_observer_windows.py b/tests/test_input_observer_windows.py index 24c7a36..cf71c9d 100644 --- a/tests/test_input_observer_windows.py +++ b/tests/test_input_observer_windows.py @@ -340,6 +340,59 @@ def test_callbacks_emit_normalized_events_filter_injected_and_chain() -> None: observer.stop() +def test_injected_transition_invalidates_an_in_flight_frame() -> None: + observer = make_observer(lambda _event: None) + observer.start() + cut = observer.begin_frame_capture() + payload = KBDLLHOOKSTRUCT( + vkCode=0x41, + scanCode=30, + flags=LLKHF_INJECTED, + ) + + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(payload), + ) + + assert not observer.finish_frame_capture(cut) + observer.complete_frame_capture(cut) + observer.stop() + + +def test_native_callback_active_before_frame_invalidates_the_cut( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = make_observer(lambda _event: None) + callback_reserved = threading.Event() + release_callback = threading.Event() + original_enqueue = observer._enqueue_input + + def hold_after_reservation(transition) -> None: + callback_reserved.set() + assert release_callback.wait(timeout=2) + original_enqueue(transition) + + monkeypatch.setattr(observer, "_enqueue_input", hold_after_reservation) + observer.start() + payload = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=30) + callback_thread = threading.Thread( + target=observer._keyboard_hook_callback, + args=(HC_ACTION, WM_KEYDOWN, ctypes.addressof(payload)), + ) + callback_thread.start() + assert callback_reserved.wait(timeout=2) + + cut = observer.begin_frame_capture() + release_callback.set() + callback_thread.join(timeout=2) + assert not callback_thread.is_alive() + assert not observer.finish_frame_capture(cut) + observer.complete_frame_capture(cut) + observer.stop() + + def test_partial_setup_permission_failure_rolls_back_installed_hook() -> None: user32 = FakeUser32( hook_results=[501, 0], @@ -728,6 +781,60 @@ def invoke_key_hook() -> None: observer.stop() +def test_keyboard_hook_reserves_receipt_before_translation_queue() -> None: + translation_entered = threading.Event() + translation_release = threading.Event() + user32 = FakeUser32( + translation_entered=translation_entered, + translation_release=translation_release, + ) + reserved_timestamps = [] + events = [] + + class Receipt: + finished = False + + def fail(self, _error) -> None: + self.finished = True + + def consume(_event) -> None: + raise AssertionError("reserved input must use the receipt consumer") + + def reserve(timestamp): + reserved_timestamps.append(timestamp) + return Receipt() + + def deliver(event, receipt) -> None: + events.append(event) + receipt.finished = True + + setattr(consume, "_openadapt_input_receipt", reserve) + setattr(consume, "_openadapt_input_delivery", deliver) + observer = make_observer( + consume, + user32=user32, + clock=lambda: 111.25, + ) + observer.start() + key = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(key), + ) + == 73 + ) + assert translation_entered.wait(timeout=1) + assert reserved_timestamps == [111.25] + assert events == [] + + translation_release.set() + assert wait_until(lambda: len(events) == 1) + observer.stop() + + def test_translation_queue_overflow_fails_loud_and_unhooks() -> None: translation_entered = threading.Event() translation_release = threading.Event() diff --git a/tests/test_processing.py b/tests/test_processing.py index b3a3756..825af05 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -247,6 +247,73 @@ def test_separate_clicks_too_far_apart(self): assert len(result) == 2 assert all(isinstance(r, MouseClickEvent) for r in result) + def test_same_timestamp_does_not_skip_an_unrelated_event(self): + down = MouseDownEvent( + timestamp=1.0, + source_ordinal=1, + x=100.0, + y=100.0, + button=MouseButton.LEFT, + ) + unrelated = KeyDownEvent( + timestamp=1.1, + source_ordinal=2, + key_char="a", + ) + up = MouseUpEvent( + timestamp=1.1, + source_ordinal=3, + x=100.0, + y=100.0, + button=MouseButton.LEFT, + ) + + result = merge_consecutive_mouse_click_events([down, unrelated, up]) + + assert len(result) == 2 + assert isinstance(result[0], MouseClickEvent) + assert result[1] is unrelated + + def test_same_timestamp_clicks_use_source_identity(self): + events = [ + MouseDownEvent( + timestamp=1.0, + source_ordinal=1, + x=100.0, + y=100.0, + button=MouseButton.LEFT, + ), + MouseUpEvent( + timestamp=1.1, + source_ordinal=2, + x=100.0, + y=100.0, + button=MouseButton.LEFT, + ), + MouseDownEvent( + timestamp=1.0, + source_ordinal=3, + x=200.0, + y=200.0, + button=MouseButton.RIGHT, + ), + MouseUpEvent( + timestamp=1.1, + source_ordinal=4, + x=200.0, + y=200.0, + button=MouseButton.RIGHT, + ), + ] + + result = merge_consecutive_mouse_click_events(events) + + assert len(result) == 2 + assert [event.button for event in result] == [ + MouseButton.LEFT, + MouseButton.RIGHT, + ] + class TestDetectDragEvents: """Tests for detect_drag_events.""" diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 71a9203..2af98bf 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -13,6 +13,7 @@ OPENADAPT_WINDOW_SMOKE_OWNER=Parallels pytest tests/test_window_capture.py -m slow """ +import multiprocessing import os import queue import sys @@ -24,16 +25,20 @@ import pytest from PIL import Image +import openadapt_capture.recorder as recorder_module import openadapt_capture.window_capture as window_capture_module from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud from openadapt_capture.desktop_capture import DesktopCaptureScope from openadapt_capture.events import WindowCaptureStateV2, window_geometry_epoch_sha256 +from openadapt_capture.input_observer import ObservedMouseButton, ThreadedInputObserver from openadapt_capture.recorder import ( Event, + NativeInputFrameBoundary, OrderedEventJournal, Recorder, WindowScopedFrame, + read_input_events, read_screen_events, ) from openadapt_capture.window_capture import ( @@ -156,6 +161,348 @@ def reserve_action(): assert frame.data.geometry_generation == 2 +def test_native_receipt_snapshot_does_not_call_window_or_topology_apis( + scope, + monkeypatch, +): + image, _ = scope.capture_frame(publish=False) + generation = scope.current_generation() + journal = OrderedEventJournal() + journal.commit_window_frame( + Event( + 1.0, + "screen", + WindowScopedFrame( + image=image, + window_event_data=scope.window_event_data(), + geometry_generation=generation, + ), + ), + scope, + generation, + ) + + def forbidden_io(*_args, **_kwargs): + pytest.fail("native receipt reservation performed live window or topology I/O") + + monkeypatch.setattr(scope, "resolve", forbidden_io) + monkeypatch.setattr(scope, "_assert_display_topology", forbidden_io) + + receipt = journal.reserve_window_action_receipt(2.0, scope) + assert receipt.source_ordinal == 2 + receipt.fail(RuntimeError("test cleanup")) + + +def test_screen_reader_discards_a_frame_with_concurrent_native_input( + scope, + monkeypatch, +): + boundary = NativeInputFrameBoundary() + clean_results = iter((False, True)) + completed_tokens: list[object] = [] + + class CheckedTerminate(threading.Event): + def wait(self, timeout=None): + if timeout is not None: + assert completed_tokens + return super().wait(timeout) + + terminate = CheckedTerminate() + + class FakeObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return next(clean_results) + + def complete_frame_capture(self, token: object) -> None: + completed_tokens.append(token) + + boundary.attach(FakeObserver()) + capture_calls = 0 + original_capture = scope.capture_frame + + def counted_capture(*, publish=True): + nonlocal capture_calls + capture_calls += 1 + result = original_capture(publish=publish) + if capture_calls == 2: + terminate.set() + return result + + monkeypatch.setattr(scope, "capture_frame", counted_capture) + journal = OrderedEventJournal() + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + input_frame_boundary=boundary, + ) + + assert capture_calls == 2 + assert len(completed_tokens) == 2 + frame = journal.get_nowait() + assert frame.type == "screen" + assert frame.source_ordinal == 1 + with pytest.raises(queue.Empty): + journal.get_nowait() + + +def test_terminal_frame_seals_native_input_before_commit(scope, monkeypatch): + terminate = threading.Event() + terminate.set() + input_finished = threading.Event() + input_finished.set() + terminal_finished = threading.Event() + boundary = NativeInputFrameBoundary() + sealed = False + completed = False + + class FakeObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return True + + def seal_frame_capture(self, _token: object) -> None: + nonlocal sealed + sealed = True + + def complete_frame_capture(self, _token: object) -> None: + nonlocal completed + completed = True + + boundary.attach(FakeObserver()) + journal = OrderedEventJournal() + original_commit = journal.commit_window_frame + + def checked_commit(*args, **kwargs): + assert sealed + return original_commit(*args, **kwargs) + + monkeypatch.setattr(journal, "commit_window_frame", checked_commit) + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + input_finished=input_finished, + input_frame_boundary=boundary, + terminal_frame_finished=terminal_finished, + ) + + assert terminal_finished.is_set() + assert completed + assert journal.get_nowait().type == "screen" + + +def test_terminal_frame_retries_an_input_dirty_capture_before_signaling( + scope, + monkeypatch, +): + monkeypatch.setattr(recorder_module.config, "SCREEN_CAPTURE_FPS", 0) + terminate = threading.Event() + terminate.set() + input_finished = threading.Event() + input_finished.set() + terminal_finished = threading.Event() + boundary = NativeInputFrameBoundary() + clean_results = iter((False, True)) + seals = 0 + completes = 0 + + class FakeObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return next(clean_results) + + def seal_frame_capture(self, _token: object) -> None: + nonlocal seals + seals += 1 + + def complete_frame_capture(self, _token: object) -> None: + nonlocal completes + completes += 1 + + boundary.attach(FakeObserver()) + journal = OrderedEventJournal() + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + input_finished=input_finished, + input_frame_boundary=boundary, + terminal_frame_finished=terminal_finished, + ) + + assert terminal_finished.is_set() + assert seals == 1 + assert completes == 2 + assert journal.get_nowait().type == "screen" + + +def test_terminal_frame_deadline_does_not_signal_without_a_clean_cut( + scope, + monkeypatch, +): + monkeypatch.setattr(recorder_module.config, "SCREEN_CAPTURE_FPS", 0) + monkeypatch.setattr(recorder_module, "TERMINAL_FRAME_SEAL_TIMEOUT_SECONDS", 0.0) + terminate = threading.Event() + terminate.set() + terminal_finished = threading.Event() + boundary = NativeInputFrameBoundary() + seals = 0 + completes = 0 + + class DirtyObserver: + def begin_frame_capture(self) -> object: + return object() + + def finish_frame_capture(self, _token: object) -> bool: + return False + + def seal_frame_capture(self, _token: object) -> None: + nonlocal seals + seals += 1 + + def complete_frame_capture(self, _token: object) -> None: + nonlocal completes + completes += 1 + + boundary.attach(DirtyObserver()) + journal = OrderedEventJournal() + + with pytest.raises(WindowCaptureError, match="terminal-frame deadline"): + read_screen_events( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + threading.Event(), + window_scope=scope, + input_frame_boundary=boundary, + terminal_frame_finished=terminal_finished, + ) + + assert not terminal_finished.is_set() + assert seals == 0 + assert completes == 1 + with pytest.raises(queue.Empty): + journal.get_nowait() + + +def test_processor_binds_first_later_frame_as_the_exact_action_after(scope): + journal = queue.Queue() + frames = [] + for timestamp, ordinal in ((1.0, 1), (3.0, 3)): + image, _ = scope.capture_frame(publish=False) + frames.append( + Event( + timestamp, + "screen", + WindowScopedFrame( + image=image, + window_event_data=scope.window_event_data(), + geometry_generation=scope.current_generation(), + ), + ordinal, + ) + ) + journal.put(frames[0]) + journal.put( + Event( + 2.0, + "action", + { + "name": "click", + "mouse_x": 1.0, + "mouse_y": 2.0, + "mouse_button_name": "left", + "mouse_pressed": True, + "window_geometry_generation": 1, + }, + 2, + ) + ) + journal.put(frames[1]) + queues = [queue.Queue() for _ in range(6)] + counters = [multiprocessing.Value("i", 0) for _ in range(5)] + terminate = threading.Event() + terminate.set() + + recorder_module.process_events( + journal, + queues[0], + queues[1], + queues[2], + queues[3], + queues[4], + queues[5], + SimpleNamespace(timestamp=0.0), + terminate, + threading.Event(), + *counters, + ) + + action = queues[1].get_nowait() + assert action.data["screenshot_source_ordinal"] == 1 + assert action.data["after_screenshot_source_ordinal"] == 3 + assert action.data["after_window_event_source_ordinal"] == 3 + assert action.data["after_window_geometry_generation"] == 1 + + +def test_processor_refuses_a_native_action_without_a_terminal_after_frame(scope): + image, _ = scope.capture_frame(publish=False) + journal = queue.Queue() + journal.put( + Event( + 1.0, + "screen", + WindowScopedFrame( + image=image, + window_event_data=scope.window_event_data(), + geometry_generation=scope.current_generation(), + ), + 1, + ) + ) + journal.put( + Event( + 2.0, + "action", + {"name": "press", "key_char": "a", "window_geometry_generation": 1}, + 2, + ) + ) + queues = [queue.Queue() for _ in range(6)] + counters = [multiprocessing.Value("i", 0) for _ in range(5)] + terminate = threading.Event() + terminate.set() + + with pytest.raises(WindowCaptureError, match="pending actions received an after frame"): + recorder_module.process_events( + journal, + queues[0], + queues[1], + queues[2], + queues[3], + queues[4], + queues[5], + SimpleNamespace(timestamp=0.0), + terminate, + threading.Event(), + *counters, + ) + + def test_input_observation_precedes_an_in_flight_window_frame(fake): capture_entered = threading.Event() release_capture = threading.Event() @@ -254,6 +601,140 @@ def reserve_action(): assert journal.get_nowait().type == "screen" +def test_native_receipt_reserves_before_async_input_delivery(scope, monkeypatch): + """A delayed observer callback cannot bind a post-input frame.""" + + class ReadyObserver(ThreadedInputObserver): + def __init__(self, callback): + super().__init__( + callback, + observe_keyboard=False, + observe_mouse=True, + capture_mouse_moves=True, + shutdown_timeout=1.0, + ) + self.release_loop = threading.Event() + + def _setup(self): + return + + def _run_loop(self): + self.release_loop.wait() + + def _teardown(self): + return + + def _wake(self): + self.release_loop.set() + + journal = OrderedEventJournal() + first_image, _ = scope.capture_frame(publish=False) + first_generation = scope.current_generation() + journal.commit_window_frame( + Event( + 1.0, + "screen", + WindowScopedFrame( + image=first_image, + window_event_data=scope.window_event_data(), + geometry_generation=first_generation, + ), + ), + scope, + first_generation, + ) + + observation_entered = threading.Event() + release_observation = threading.Event() + + class BlockingStructuralObserver: + def open_current_thread(self): + return + + def close_current_thread(self): + return + + def observe(self, _request): + observation_entered.set() + assert release_observation.wait(timeout=5) + return None + + observers = [] + + def create_observer(callback, **_kwargs): + observer = ReadyObserver(callback) + observers.append(observer) + return observer + + monkeypatch.setattr(recorder_module, "create_input_observer", create_observer) + terminate = threading.Event() + started = threading.Event() + input_reader = threading.Thread( + target=read_input_events, + args=( + journal, + terminate, + SimpleNamespace(timestamp=time.time()), + started, + ), + kwargs={ + "coordinate_scope": scope, + "structural_observer": BlockingStructuralObserver(), + }, + ) + input_reader.start() + assert started.wait(timeout=5) + observer = observers[0] + native_event = ObservedMouseButton( + x=310.0, + y=170.0, + button="left", + pressed=True, + timestamp=1.5, + ) + receipt = observer._reserve_receipt(native_event.timestamp) + observer._emit( + native_event, + receipt=receipt, + ) + assert observation_entered.wait(timeout=5) + + later_image, _ = scope.capture_frame(publish=False) + later_generation = scope.current_generation() + journal.commit_window_frame( + Event( + 2.0, + "screen", + WindowScopedFrame( + image=later_image, + window_event_data=scope.window_event_data(), + geometry_generation=later_generation, + ), + ), + scope, + later_generation, + ) + release_observation.set() + deadline = time.monotonic() + 5 + while not bool(getattr(receipt, "finished", False)): + if time.monotonic() >= deadline: + pytest.fail("native receipt was not completed") + time.sleep(0.001) + terminate.set() + input_reader.join(timeout=5) + assert not input_reader.is_alive() + + initial = journal.get_nowait() + action = journal.get_nowait() + later = journal.get_nowait() + assert [(initial.type, initial.source_ordinal), (action.type, action.source_ordinal)] == [ + ("screen", 1), + ("action", 2), + ] + assert (later.type, later.source_ordinal) == ("screen", 3) + assert action.data["window_geometry_generation"] == first_generation + + def test_window_capture_state_rejects_scales_not_derived_from_content(scope): scope.capture_frame() state = scope.window_event_data()["state"]