From e3db4c78a8e7ca2eb7e344aa853a6ce11d46ca47 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:28:43 -0700 Subject: [PATCH 01/17] feat(protocol): record a computer's display, from the driver's @record stream or by polling @capture computer-recording/v1 in ComputerDriverEnvironment. A driver declares `recording = Recording()` to serve it; the base class owns each recording's life: start and end with their retries, max_concurrent, max_duration_s and a free-disk floor, segments rolled over at max_segment_s, on a size change or after a gap, uploads through the start's write_namespace grant as each segment closes, a fresh grant at end for any that missed, and the manifest. The pixels come from a new @record operation, an async iterator of RecordedFrame (frames the OS streams, which the base class encodes as H.264 with each frame's captured_at as its timestamp) or RecordedSegment (files a native recorder finished). A driver without @record falls back to the base class polling @capture, with a pointer drawn at @pointer_position, capped at 15 fps. PyAV comes in a new `recording` extra; start without it is refused unsupported. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/pyproject.toml | 5 + .../agentenv_protocol/computer/__init__.py | 7 +- .../src/agentenv_protocol/computer/card.py | 6 + .../src/agentenv_protocol/computer/checks.py | 4 + .../src/agentenv_protocol/computer/driver.py | 4 + .../agentenv_protocol/computer/extensions.py | 18 +- .../agentenv_protocol/computer/operations.py | 4 +- .../agentenv_protocol/computer/recording.py | 413 ++++++++++++++++++ .../src/agentenv_protocol/computer/runtime.py | 7 + .../tests/test_computer_recording.py | 365 ++++++++++++++++ uv.lock | 75 +++- 11 files changed, 902 insertions(+), 6 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py create mode 100644 packages/agentenv-protocol/tests/test_computer_recording.py diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index cf07119b4..8e12e00a9 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -41,6 +41,10 @@ computer = [ "Pillow>=10.4", "uvicorn>=0.30", ] +# A computer driver that records: computer-recording/v1 encodes H.264 with PyAV. +recording = [ + "av>=17", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -50,6 +54,7 @@ dev = [ "mcp>=1.25,<2", "fastmcp>=3.4,<4", # the standalone-fastmcp mount path "Pillow>=10.4", # the computer extra's frame crops and encodings + "av>=17", # the recording extra's encoder, and the tests' decoder ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 812c466fd..de519e2d0 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -25,6 +25,7 @@ COMPUTER_KEYBOARD, COMPUTER_POINTER, COMPUTER_READY, + COMPUTER_RECORDING, COMPUTER_SPACES, ) from .operations import ( @@ -44,6 +45,7 @@ pointer_position, pointer_up, press, + record, run_in_session, scroll, set_display, @@ -51,6 +53,7 @@ write_files, ) from .platform import Platform +from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording from .settings import SETTINGS_ENV, ComputerSettings __all__ = [ @@ -58,10 +61,10 @@ "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", - "run_in_session", "write_files", + "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed", "Internal", "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD", - "COMPUTER_EXEC", + "COMPUTER_EXEC", "COMPUTER_RECORDING", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py index 671af965c..f07319d4a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING +from .recording import recording_params from ..types import METHOD_ADD, EnvironmentExtension, EnvironmentTool, IntakeDeclaration, IntakeFormat, intake_extension if TYPE_CHECKING: @@ -17,6 +18,7 @@ COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" +COMPUTER_RECORDING = "urn:agentenv:computer-recording/v1" MAX_EXEC_TIMEOUT_S = 3600 KEYBOARD_RANGES = {"tap_hold_ms": (0, 1000), "type_delay_ms": (0, 1000), "max_hold_ms": (0, 600_000)} @@ -48,6 +50,8 @@ def served_extensions(driver: ComputerDriverEnvironment, need = _EXTENSION_NEEDS.get(extension.uri) if need is not None and not driver.has(need): continue + if extension.uri == COMPUTER_RECORDING and driver.recording is None: + continue dropped = {op for (uri, op), need in _METHOD_NEEDS.items() if uri == extension.uri and not driver.has(need)} if dropped: methods = {op: entry for op, entry in extension.params["methods"].items() if op not in dropped} @@ -70,6 +74,8 @@ def extension_params(driver: ComputerDriverEnvironment, uri: str) -> dict: "layouts": list(driver.layouts)} if uri == COMPUTER_EXEC: return {"max_output_bytes": driver.settings.exec_output_bytes, "max_timeout_s": MAX_EXEC_TIMEOUT_S} + if uri == COMPUTER_RECORDING and driver.recording is not None: + return recording_params(driver.recording, driver.has("record")) return {} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py index c65c619e9..a9eef98e6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -5,6 +5,7 @@ from ..errors import AgentEnvError from .keys import canonical from .operations import declared +from .recording import check_recording BUTTONS = ("left", "right", "middle", "back", "forward") @@ -38,5 +39,8 @@ def check_features(cls: type) -> None: problems.append(f"keys: {e.message}") if ("set_display" in declared(cls)) != bool(cls.display_modes): problems.append("display_modes and @set_display come together") + problems += check_recording(cls.recording) + if "record" in declared(cls) and cls.recording is None: + problems.append("@record needs recording = Recording()") if problems: raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 1181505ed..df9c76807 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -14,6 +14,7 @@ from .extensions import Extensions from .operations import ReadyCheck, check_class, default_for from .platform import Platform, detect +from .recording import Recording from .runtime import ComputerRuntime from .settings import ComputerSettings, load_settings from .tools import Tools @@ -42,6 +43,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen layouts: ClassVar[tuple[str, ...]] = ("us",) #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``. supports: ClassVar[list] = [] + #: ``Recording()`` serves computer-recording/v1: from ``@record`` if the driver has one, else by polling + #: ``@capture``. + recording: ClassVar[Recording | None] = None tool_access = True #: What the base class keeps for the driver, built by ``__init__``. _computer: ComputerRuntime | None = None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index 4ce812396..b69aefd28 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -1,5 +1,5 @@ -"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings -and commands in the session.""" +"""The framework extensions agentenv calls, never agents: readiness, display, coordinate spaces, keyboard settings, +commands in the session and recordings.""" from __future__ import annotations from typing import TYPE_CHECKING, Callable, Optional @@ -11,6 +11,7 @@ COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_READY, + COMPUTER_RECORDING, COMPUTER_SPACES, KEYBOARD_RANGES, MAX_EXEC_TIMEOUT_S, @@ -111,6 +112,19 @@ async def _computer_exec_run(self, argv: list, request_id: str, cwd: Optional[st cap = self.settings.exec_output_bytes return await self._computer.command(request_id, lambda: _run(self._computer, cap, argv, cwd, env, timeout)) + @extension(COMPUTER_RECORDING, op="start", + description="Record the display: start returns a recording whose segments upload through its grant as " + "they close, and end seals it and returns its manifest; repeats answer the same.") + async def _computer_recording_start(self, request_id: str, scope: str = "display", window: Optional[str] = None, + fps: Optional[int] = None, max_edge: Optional[int] = None, cursor: bool = True, + upload: Optional[dict] = None) -> dict: + return await self._computer.recordings.start(request_id, scope, window, fps, max_edge, cursor, upload) + + @extension(COMPUTER_RECORDING, op="end") + async def _computer_recording_end(self, recording_id: str, upload: Optional[dict] = None, + reason: str = "end") -> dict: + return await self._computer.recordings.end(recording_id, upload, reason) + async def _run(computer: ComputerRuntime, cap: int, argv: list, cwd: Optional[str], env: Optional[dict], timeout: float) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index 0f7419da6..1e9936ec1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -11,6 +11,7 @@ pointer_move(x, y), pointer_down(button), pointer_up(button), pointer_position() -> (x, y) key_down(keys), key_up(keys) run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) + record(hints) -> an async iterator of RecordedFrame or RecordedSegment, until it is closed """ from __future__ import annotations @@ -25,7 +26,7 @@ "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), "keyboard": ("key_down", "key_up"), } -OPTIONAL = ("set_display", "run_in_session", "write_files") +OPTIONAL = ("set_display", "run_in_session", "write_files", "record") INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "key_down", "key_up"}) @@ -114,6 +115,7 @@ def deco(f: Callable) -> Callable: key_up = _marker("key_up") run_in_session = _marker("run_in_session") write_files = _marker("write_files") +record = _marker("record") def default_for(operation: str, **predicates: Any) -> Callable: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py new file mode 100644 index 000000000..d360ec017 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -0,0 +1,413 @@ +"""Recording a computer's display, computer-recording/v1: the base class runs each recording from the driver's +``@record`` stream, or by polling ``@capture`` where the driver has none, encodes frames as H.264 segments, uploads +each segment as it closes and seals the recording at ``end`` with its manifest. Encoding needs the ``recording`` +extra (PyAV).""" +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import hashlib +import shutil +import time +import uuid +from collections import OrderedDict +from collections.abc import AsyncIterator +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timezone +from fractions import Fraction +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + +from PIL import Image as PILImage +from PIL import ImageDraw +from pydantic import ValidationError + +from ..transfers import NamespaceUploader, WriteNamespaceGrant +from .errors import InvalidArgument, LimitExceeded, Unsupported +from .frames import to_pil + +if TYPE_CHECKING: + from .runtime import ComputerRuntime + +MP4 = "video/mp4; codecs=avc1" +#: Where segments are written, relative to the session user's home, and kept when there is no grant. +RECORDINGS_DIR = ".agentenv-recordings" +SCOPES = ("display", "window") +REASONS = ("end", "reset", "teardown") +DEFAULT_FPS = 10 +#: The fallback polls ``@capture``, a full read of the screen each pass, so it is capped lower than streams. +MAX_POLLED_FPS = 15 +#: Free disk below this ends a recording (``limit``). +MIN_FREE_BYTES = 512 << 20 +#: A stream that keeps failing this long ends its recording (``error``). +FAILING_S = 60.0 +#: Frames further apart than this start a new segment. +GAP_S = 2.0 +KEEP_ENDED = 64 + + +@dataclass(frozen=True) +class Recording: + """A driver's recording features, as a class attribute: ``recording = Recording()`` serves computer-recording/v1.""" + max_fps: int = 15 + max_concurrent: int = 2 + max_duration_s: int = 3600 + max_segment_s: int = 300 + + +@dataclass(frozen=True) +class RecordHints: + """What ``@record`` is asked for. A stream that yields segments writes its files under ``directory`` and rolls + them over at ``max_segment_s``; one that yields frames leaves both to the base class.""" + scope: str + fps: int + max_edge: Optional[int] + cursor: bool + max_segment_s: int + directory: Path + + +@dataclass(frozen=True) +class RecordedFrame: + """A frame of a stream: a Pillow image or a PyAV ``VideoFrame``, and when it was captured.""" + image: Any + captured_at: datetime + + +@dataclass(frozen=True) +class RecordedSegment: + """A finished segment a native recorder wrote, and the times of its first and last frames.""" + path: Path + started_at: datetime + ended_at: datetime + width: int + height: int + frames: int + format: str = MP4 + + +def check_recording(recording: object) -> list[str]: + if recording is None: + return [] + if not isinstance(recording, Recording): + return [f"recording is a Recording, got {type(recording).__name__}"] + problems = [] + for name, low, high in (("max_fps", 1, 60), ("max_concurrent", 1, 16), ("max_duration_s", 1, 86_400), + ("max_segment_s", 1, 3600)): + value = getattr(recording, name) + if not isinstance(value, int) or not low <= value <= high: + problems.append(f"recording.{name} is an integer from {low} to {high}, got {value!r}") + return problems + + +def recording_params(recording: Recording, streams: bool) -> dict: + """The extension's params on the card: what ``start`` can be asked for, and the limits.""" + most = recording.max_fps if streams else min(recording.max_fps, MAX_POLLED_FPS) + return {"formats": [MP4], "scopes": ["display"], "fps": {"default": min(DEFAULT_FPS, most), "max": most}, + "cursor": True, "source": "stream" if streams else "capture", "max_segment_s": recording.max_segment_s, + "max_concurrent": recording.max_concurrent, "max_duration_s": recording.max_duration_s} + + +class Recordings: + """The recordings of one driver: started by request id, open ones counted against ``max_concurrent``, ended + ones kept with their manifests so a repeated ``end`` answers the same.""" + + def __init__(self, runtime: ComputerRuntime) -> None: + self.runtime = runtime + self.running: dict[str, _Recording] = {} + self.ended: OrderedDict[str, dict] = OrderedDict() + self.started: OrderedDict[str, dict] = OrderedDict() + self._lock = asyncio.Lock() + + async def start(self, request_id: str, scope: str, window: Optional[str], fps: Optional[int], + max_edge: Optional[int], cursor: bool, upload: Optional[dict]) -> dict: + features: Recording = self.runtime.driver.recording + if not (isinstance(request_id, str) and request_id): + raise InvalidArgument("request_id is a non-empty string") + async with self._lock: + if request_id in self.started: + return self.started[request_id] + hints, grant = self._hints(features, scope, window, fps, max_edge, cursor, upload) + if sum(not r.done for r in self.running.values()) >= features.max_concurrent: + raise LimitExceeded(f"{features.max_concurrent} recordings are running, the most this computer " + "records at once; end one first") + try: + import av # noqa: F401 + except ImportError: + raise Unsupported("recording needs PyAV: install agentenv-framework-protocol[computer,recording]") \ + from None + recording_id = f"rec-{uuid.uuid4().hex[:12]}" + root = Path.home() / RECORDINGS_DIR + hints = dataclasses.replace(hints, directory=root / recording_id) + hints.directory.mkdir(parents=True, exist_ok=True) + recording = _Recording(recording_id, hints, features, root, grant, self.runtime) + self.running[recording_id] = recording + recording.task = asyncio.create_task(recording.run()) + answer = {"recording_id": recording_id, "started_at": _iso(recording.requested_at), "format": MP4} + self.started[request_id] = answer + while len(self.started) > 4 * KEEP_ENDED: + self.started.popitem(last=False) + return answer + + async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> dict: + if reason not in REASONS: + raise InvalidArgument(f"reason is one of {list(REASONS)}, got {reason!r}") + if recording_id in self.ended: + return self.ended[recording_id] + recording = self.running.get(recording_id) + if recording is None: + raise InvalidArgument(f"there is no recording {recording_id!r} on this computer") + grant = _grant(upload) if upload is not None else None + manifest = await recording.end(reason, grant) + self.running.pop(recording_id, None) + self.ended[recording_id] = manifest + while len(self.ended) > KEEP_ENDED: + self.ended.popitem(last=False) + return manifest + + def _hints(self, features: Recording, scope: str, window: Optional[str], fps: Optional[int], + max_edge: Optional[int], cursor: bool, upload: Optional[dict]) -> tuple[RecordHints, Any]: + if scope not in SCOPES: + raise InvalidArgument(f"scope is 'display' or 'window', got {scope!r}") + if scope == "window" or window is not None: + raise Unsupported("this computer records the whole display only; window recordings need computer-window/v1") + params = recording_params(features, self.runtime.has("record")) + fps = params["fps"]["default"] if fps is None else fps + if not (isinstance(fps, int) and 1 <= fps <= params["fps"]["max"]): + raise InvalidArgument(f"fps is an integer from 1 to {params['fps']['max']}, got {fps!r}") + if max_edge is not None and not (isinstance(max_edge, int) and 64 <= max_edge <= 8192): + raise InvalidArgument(f"max_edge is an integer from 64 to 8192, got {max_edge!r}") + grant = _grant(upload) if upload is not None else None + return RecordHints(scope, fps, max_edge, bool(cursor), features.max_segment_s, Path()), grant + + +class _Recording: + def __init__(self, recording_id: str, hints: RecordHints, features: Recording, root: Path, + grant: Optional[WriteNamespaceGrant], runtime: ComputerRuntime) -> None: + self.id = recording_id + self.hints = hints + self.features = features + self.root = root + self.runtime = runtime + self.uploader = NamespaceUploader(grant) if grant is not None else None + self.requested_at = datetime.now(timezone.utc) + self.streams = runtime.has("record") + self.cursor = hints.cursor if self.streams else hints.cursor and runtime.has("pointer") + self.segments: list[dict] = [] + self.encoder: Optional[_Encoder] = None + self.ended_by: Optional[str] = None + self.error: Optional[str] = None + self.done = False + self.task: Optional[asyncio.Task] = None + self.uploads: set[asyncio.Task] = set() + self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"computer-{recording_id}") + self._last_disk_check = 0.0 + + async def run(self) -> None: + failing_since: Optional[float] = None + try: + while True: + try: + async for item in self._source(): + failing_since = None + if isinstance(item, RecordedSegment): + await self._add_segment(item) + else: + await self._add_frame(item) + if self.ended_by is not None: + return + except asyncio.CancelledError: + raise + except Exception as e: + failing_since = failing_since or time.monotonic() + self.error = f"{type(e).__name__}: {e}" + if time.monotonic() - failing_since > FAILING_S: + self.ended_by = "error" + return + await self._close() + await asyncio.sleep(1.0) + finally: + await self._close() + self.done = True + + def _source(self) -> AsyncIterator: + if self.streams: + return self.runtime.stream(self.hints) + return _poll(self.runtime, self.hints, self.cursor) + + async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: + if self.task is not None and not self.task.done(): + self.task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self.task + if self.ended_by is None: + self.ended_by = reason + if self.uploads: + await asyncio.gather(*self.uploads, return_exceptions=True) + uploader = NamespaceUploader(grant) if grant is not None else self.uploader + if uploader is not None: + for segment in self.segments: + if not segment["uploaded"]: + await self._upload(uploader, segment) + self.executor.shutdown(wait=False) + with contextlib.suppress(OSError): + self.hints.directory.rmdir() + first, last = (self.segments[0], self.segments[-1]) if self.segments else ({}, {}) + manifest = {"recording_id": self.id, "scope": self.hints.scope, "format": MP4, "fps": self.hints.fps, + "cursor": self.cursor, "source": "stream" if self.streams else "capture", + "started_at": first.get("started_at"), "ended_at": last.get("ended_at"), + "ended_by": self.ended_by, "segments": [{k: v for k, v in s.items() if k != "file"} + for s in self.segments]} + if self.error and self.ended_by == "error": + manifest["error"] = self.error + return manifest + + async def _add_frame(self, item: RecordedFrame) -> None: + frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge) + at = item.captured_at + if self.encoder is not None and ((frame.width, frame.height) != self.encoder.size + or (at - self.encoder.last_at).total_seconds() > GAP_S + or (at - self.encoder.started_at).total_seconds() >= self.hints.max_segment_s): + await self._close() + if self.encoder is None: + path = self.hints.directory / f"{len(self.segments):04d}.mp4" + self.encoder = await self._in_thread(_Encoder, path, frame.width, frame.height, self.hints.fps, at) + await self._in_thread(self.encoder.add, frame, at) + self._check_limits(at) + + async def _add_segment(self, item: RecordedSegment) -> None: + index = len(self.segments) + target = self.hints.directory / f"{index:04d}.mp4" + if Path(item.path) != target: + await asyncio.to_thread(shutil.move, str(item.path), target) + await self._finish(target, item.started_at, item.ended_at, item.width, item.height, item.frames) + self._check_limits(item.ended_at) + + def _check_limits(self, at: datetime) -> None: + if (at - self.requested_at).total_seconds() >= self.features.max_duration_s: + self.ended_by = "limit" + elif time.monotonic() - self._last_disk_check > 5: + self._last_disk_check = time.monotonic() + if shutil.disk_usage(self.root).free < MIN_FREE_BYTES: + self.ended_by = "limit" + + async def _close(self) -> None: + encoder, self.encoder = self.encoder, None + if encoder is None: + return + await self._in_thread(encoder.close) + if encoder.frames: + await self._finish(encoder.path, encoder.started_at, encoder.last_at, *encoder.size, encoder.frames) + else: + encoder.path.unlink(missing_ok=True) + + async def _finish(self, path: Path, started_at: datetime, ended_at: datetime, width: int, height: int, + frames: int) -> None: + size, digest = await asyncio.to_thread(_digest, path) + segment = {"index": len(self.segments), "path": f"{self.id}/{path.name}", "started_at": _iso(started_at), + "ended_at": _iso(ended_at), "width": width, "height": height, "frames": frames, "bytes": size, + "sha256": digest, "uploaded": False, "file": path} + self.segments.append(segment) + if self.uploader is not None: + task = asyncio.create_task(self._upload(self.uploader, segment)) + self.uploads.add(task) + task.add_done_callback(self.uploads.discard) + + async def _upload(self, uploader: NamespaceUploader, segment: dict) -> None: + try: + await uploader.upload(segment["path"], segment["file"]) + except Exception as e: + segment["upload_error"] = f"{getattr(e, 'code', type(e).__name__)}: {e}" + return + segment["uploaded"] = True + segment.pop("upload_error", None) + segment["file"].unlink(missing_ok=True) + + async def _in_thread(self, fn, *args): + return await asyncio.get_running_loop().run_in_executor(self.executor, fn, *args) + + +class _Encoder: + """One segment's H.264 encoder: each frame's timestamp is its ``captured_at`` from the segment's first frame, so + the video's timeline is the computer's clock.""" + + def __init__(self, path: Path, width: int, height: int, fps: int, started_at: datetime) -> None: + import av + + self.path = path + self.size = (width, height) + self.started_at = self.last_at = started_at + self.frames = 0 + self._last_pts = -1 + self.container = av.open(str(path), "w", format="mp4", options={"movflags": "+faststart"}) + self.stream = self.container.add_stream( + "libx264", rate=fps, options={"preset": "ultrafast", "tune": "zerolatency", "crf": "28", "g": str(2 * fps)}) + self.stream.width, self.stream.height, self.stream.pix_fmt = width, height, "yuv420p" + self.stream.codec_context.time_base = Fraction(1, 1000) + + def add(self, frame: Any, at: datetime) -> None: + pts = max(int((at - self.started_at).total_seconds() * 1000), self._last_pts + 1) + frame.pts, frame.time_base = pts, Fraction(1, 1000) + for packet in self.stream.encode(frame): + self.container.mux(packet) + self._last_pts, self.last_at = pts, at + self.frames += 1 + + def close(self) -> None: + for packet in self.stream.encode(): + self.container.mux(packet) + self.container.close() + + +async def _poll(runtime: ComputerRuntime, hints: RecordHints, cursor: bool) -> AsyncIterator[RecordedFrame]: + """The fallback stream: ``@capture`` every ``1/fps`` seconds, with a pointer drawn where the driver says it is.""" + interval, due = 1 / hints.fps, time.monotonic() + while True: + display = await runtime.display_now() + raw = await runtime.call("capture") + captured_at = datetime.now(timezone.utc) + image = to_pil(raw, display) + if cursor: + x, y = await runtime.pointer() + _draw_pointer(image, x * display.scale, y * display.scale) + yield RecordedFrame(image, captured_at) + due = max(due + interval, time.monotonic()) + await asyncio.sleep(due - time.monotonic()) + + +def _draw_pointer(image: PILImage.Image, x: float, y: float) -> None: + arrow = [(0, 0), (0, 17), (4, 13), (7, 20), (10, 19), (7, 12), (12, 12)] + ImageDraw.Draw(image).polygon([(x + dx, y + dy) for dx, dy in arrow], fill=(0, 0, 0), outline=(255, 255, 255)) + + +def _video_frame(image: Any, max_edge: Optional[int]) -> Any: + """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``.""" + import av + + frame = image if isinstance(image, av.VideoFrame) else av.VideoFrame.from_image(image.convert("RGB")) + width, height = frame.width, frame.height + if max_edge is not None and max(width, height) > max_edge: + ratio = max_edge / max(width, height) + width, height = round(width * ratio), round(height * ratio) + return frame.reformat(width=max(2, width - width % 2), height=max(2, height - height % 2), format="yuv420p") + + +def _grant(upload: object) -> WriteNamespaceGrant: + try: + return WriteNamespaceGrant.model_validate(upload) + except ValidationError: + raise InvalidArgument("upload is a write_namespace grant") from None + + +def _digest(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return path.stat().st_size, digest.hexdigest() + + +def _iso(at: datetime) -> str: + return at.isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 36223531b..d4e43bfbe 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -6,6 +6,7 @@ import asyncio import contextlib from collections import OrderedDict +from collections.abc import AsyncIterator from typing import TYPE_CHECKING, Any, Awaitable, Callable from mcp.types import CallToolResult @@ -16,6 +17,7 @@ from .frames import Frame, FrameStore, to_pil from .operations import GROUPS, INPUT, Display, ReadyCheck, resolve from .queue import InputQueue +from .recording import RecordHints, Recordings from .results import EffectData, InputResult, send if TYPE_CHECKING: @@ -37,6 +39,7 @@ def __init__(self, driver: ComputerDriverEnvironment) -> None: self._releases: dict[str, asyncio.Task] = {} self._commands_running: dict[str, asyncio.Task] = {} self._commands_finished: OrderedDict = OrderedDict() + self.recordings = Recordings(self) def has(self, capability: str) -> bool: if capability in GROUPS: @@ -56,6 +59,10 @@ async def call(self, operation: str, *args: Any) -> Any: error = DeliveryFailed if operation in INPUT else Internal raise error(f"@{operation} failed: {type(e).__name__}: {e}") from e + def stream(self, hints: RecordHints) -> AsyncIterator: + """The driver's ``@record`` stream for one recording.""" + return getattr(self.driver, self.operations["record"])(hints) + async def display_now(self) -> Display: display = await self.call("display") if not isinstance(display, Display) or display.width <= 0 or display.height <= 0 or display.scale <= 0: diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py new file mode 100644 index 000000000..44c7e0511 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -0,0 +1,365 @@ +"""computer-recording/v1: a driver with ``recording`` serves it, from its ``@record`` stream or by polling ``@capture``; +segments roll over and upload as they close, the manifest matches the files, and the limits and retries hold.""" +from __future__ import annotations + +import asyncio +import hashlib +from datetime import datetime, timedelta, timezone +from fractions import Fraction +from unittest.mock import AsyncMock, patch + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") +av = pytest.importorskip("av") + +from computer_fake import FakeDriver, MinimalDriver, http_client # noqa: E402 +from PIL import Image as PILImage # noqa: E402 + +from agentenv_protocol.computer import ( # noqa: E402 + COMPUTER_RECORDING, + DeviceUnavailable, + RecordedFrame, + RecordedSegment, + Recording, + record, +) +from agentenv_protocol.computer import recording as recording_module # noqa: E402 +from agentenv_protocol.transfers import TransferError # noqa: E402 + +START, END = "/agentenv/ext/computer-recording/start", "/agentenv/ext/computer-recording/end" + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path + + +class Recorded(FakeDriver): + recording = Recording(max_segment_s=1) + + +class Streamed(FakeDriver): + """A stream of the fake screen's frames, as a driver with its own capture stream yields them.""" + recording = Recording(max_fps=30, max_segment_s=60) + + @record + async def frames(self, hints): + while True: + yield RecordedFrame(self.screen.image.copy(), datetime.now(timezone.utc)) + await asyncio.sleep(1 / hints.fps) + + +class Segmented(FakeDriver): + """A native recorder that writes whole segments itself.""" + recording = Recording(max_segment_s=60) + + @record + async def segments(self, hints): + for n in range(2): + path = hints.directory / ("0000.mp4" if n == 0 else f"native-{n}.mp4") + start = datetime.now(timezone.utc) + await asyncio.to_thread(_mp4, path, 5, (160, 100)) + yield RecordedSegment(path, start, start + timedelta(seconds=0.4), 160, 100, 5) + await asyncio.sleep(3600) + + +def _mp4(path, frames, size): + out = av.open(str(path), "w", format="mp4") + stream = out.add_stream("libx264", rate=10) + stream.width, stream.height, stream.pix_fmt = *size, "yuv420p" + for i in range(frames): + frame = av.VideoFrame.from_image(PILImage.new("RGB", size, (i * 40, 0, 0))).reformat(format="yuv420p") + frame.pts, frame.time_base = i * 100, Fraction(1, 1000) + for p in stream.encode(frame): + out.mux(p) + for p in stream.encode(): + out.mux(p) + out.close() + + +def _grant() -> dict: + return {"root_path": "rec/run-1", "expires_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + "max_objects": 50, "max_object_bytes": 10 << 20, "max_total_bytes": 100 << 20, + "write": {"kind": "http-post-policy", "url": "https://store.test/", "fields": {}, "path_field": "key", + "file_field": "file"}} + + +async def _record(driver, seconds: float, start: dict | None = None, end: dict | None = None) -> tuple[dict, dict]: + async with http_client(driver) as http: + started = (await http.post(START, json={"request_id": "r", "fps": 10, **(start or {})})).json() + await asyncio.sleep(seconds) + manifest = (await http.post(END, json={"recording_id": started["recording_id"], **(end or {})})).json() + return started, manifest + + +def _decoded(path) -> list: + with av.open(str(path)) as container: + return list(container.decode(container.streams.video[0])) + + +def _at(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +@pytest.mark.asyncio +async def test_a_driver_with_recording_serves_it_and_one_without_does_not(): + async with http_client(Recorded()) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + async with http_client(FakeDriver()) as http: + plain = (await http.get("/.well-known/agent-env.json")).json() + extension = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) + assert set(extension["params"]["methods"]) == {"start", "end"} + assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { + "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, + "cursor": True, "source": "capture", "max_segment_s": 1, "max_concurrent": 2, "max_duration_s": 3600} + assert COMPUTER_RECORDING not in [e["uri"] for e in plain["capabilities"]["extensions"]] + + +def test_recording_features_and_record_are_checked_when_the_class_is_defined(): + with pytest.raises(TypeError, match="recording.max_fps is an integer from 1 to 60"): + class TooFast(FakeDriver): + recording = Recording(max_fps=120) + with pytest.raises(TypeError, match="@record needs recording = Recording()"): + class Undeclared(FakeDriver): + @record + async def frames(self, hints): + yield None + with pytest.raises(TypeError, match="recording is a Recording"): + class Wrong(FakeDriver): + recording = {"max_fps": 10} + + +@pytest.mark.asyncio +async def test_the_fallback_polls_capture_and_draws_the_pointer(home): + driver = Recorded() + driver.screen.pointer = (200, 120) + started, manifest = await _record(driver, 0.6) + assert started["recording_id"].startswith("rec-") and started["format"] == "video/mp4; codecs=avc1" + assert manifest["source"] == "capture" and manifest["cursor"] is True and manifest["ended_by"] == "end" + (segment,) = manifest["segments"] + file = home / ".agentenv-recordings" / segment["path"] + assert segment["uploaded"] is False and segment["bytes"] == file.stat().st_size + assert segment["sha256"] == hashlib.sha256(file.read_bytes()).hexdigest() + frames = _decoded(file) + assert len(frames) == segment["frames"] >= 4 and (frames[0].width, frames[0].height) == (1280, 800) + pixel = frames[-1].to_image().getpixel((202, 128)) + assert sum(pixel) < 200, f"no pointer drawn at the pointer: {pixel}" + assert _at(manifest["started_at"]) <= _at(manifest["ended_at"]) + + +@pytest.mark.asyncio +async def test_a_driver_without_the_pointer_group_records_without_a_cursor(home): + class Plain(MinimalDriver): + recording = Recording() + + _, manifest = await _record(Plain(), 0.4) + assert manifest["cursor"] is False and manifest["segments"][0]["width"] == 640 + + +@pytest.mark.asyncio +async def test_segments_roll_over_with_no_gap_between_them(home): + _, manifest = await _record(Recorded(), 2.6) + segments = manifest["segments"] + assert len(segments) >= 2 and [s["index"] for s in segments] == list(range(len(segments))) + for before, after in zip(segments, segments[1:]): + gap = (_at(after["started_at"]) - _at(before["ended_at"])).total_seconds() + assert 0 < gap < 0.25, gap + for segment in segments: + assert (_at(segment["ended_at"]) - _at(segment["started_at"])).total_seconds() <= 1.0 + + +@pytest.mark.asyncio +async def test_a_size_change_starts_a_segment_of_the_new_size(home): + driver = Recorded() + driver.recording = Recording(max_segment_s=60) + async with http_client(driver) as http: + started = (await http.post(START, json={"request_id": "r", "fps": 10})).json() + await asyncio.sleep(0.4) + driver.screen.resize(640, 400, 1.0) + await asyncio.sleep(0.4) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + assert [(s["width"], s["height"]) for s in manifest["segments"]] == [(1280, 800), (640, 400)] + + +@pytest.mark.asyncio +async def test_captures_that_fail_leave_a_gap_and_recording_resumes(home): + driver = Recorded() + driver.recording = Recording(max_segment_s=60) + async with http_client(driver) as http: + started = (await http.post(START, json={"request_id": "r", "fps": 10})).json() + await asyncio.sleep(0.4) + driver.fail["capture"] = DeviceUnavailable("the X server went away") + await asyncio.sleep(1.6) + manifest = (await http.post(END, json={"recording_id": started["recording_id"]})).json() + assert len(manifest["segments"]) == 2 and manifest["ended_by"] == "end" + gap = (_at(manifest["segments"][1]["started_at"]) - _at(manifest["segments"][0]["ended_at"])).total_seconds() + assert gap >= 0.9 + + +@pytest.mark.asyncio +async def test_a_stream_that_keeps_failing_ends_the_recording_with_its_error(home, monkeypatch): + monkeypatch.setattr(recording_module, "FAILING_S", 0.5) + + class Broken(FakeDriver): + recording = Recording() + + @record + async def frames(self, hints): + raise RuntimeError("no display") + yield + + _, manifest = await _record(Broken(), 2.5) + assert manifest["ended_by"] == "error" and manifest["error"] == "RuntimeError: no display" + assert manifest["segments"] == [] and manifest["started_at"] is None + + +@pytest.mark.asyncio +async def test_a_stream_of_frames_keeps_each_frames_capture_time(home): + _, manifest = await _record(Streamed(), 0.8, start={"fps": 20, "max_edge": 640}) + (segment,) = manifest["segments"] + assert manifest["source"] == "stream" and (segment["width"], segment["height"]) == (640, 400) + frames = _decoded(home / ".agentenv-recordings" / segment["path"]) + span = (_at(segment["ended_at"]) - _at(segment["started_at"])).total_seconds() + assert len(frames) == segment["frames"] >= 10 and abs(frames[-1].time - span) < 0.002 + + +@pytest.mark.asyncio +async def test_a_native_recorders_segments_are_kept_as_they_come(home): + _, manifest = await _record(Segmented(), 1.0) + assert [(s["index"], s["path"].rsplit("/", 1)[1], s["frames"]) for s in manifest["segments"]] == [ + (0, "0000.mp4", 5), (1, "0001.mp4", 5)] + assert len(_decoded(home / ".agentenv-recordings" / manifest["segments"][1]["path"])) == 5 + + +@pytest.mark.asyncio +async def test_segments_upload_as_they_close_and_end_retries_with_a_fresh_grant(home): + calls = [] + + def uploader(grant): + fail = grant.root_path == "rec/run-1" and not calls + upload = AsyncMock(side_effect=TransferError("transfer_unavailable", "down") if fail else None) + calls.append((grant.root_path, upload)) + return type("Uploader", (), {"upload": upload})() + + with patch.object(recording_module, "NamespaceUploader", side_effect=uploader): + _, manifest = await _record(Recorded(), 0.5, start={"upload": _grant()}, + end={"upload": {**_grant(), "root_path": "rec/run-2"}}) + (segment,) = manifest["segments"] + assert segment["uploaded"] is True and "upload_error" not in segment + assert [root for root, _ in calls] == ["rec/run-1", "rec/run-2"] + assert calls[1][1].await_args.args[0] == segment["path"] + assert not (home / ".agentenv-recordings" / segment["path"]).exists() + + +@pytest.mark.asyncio +async def test_an_upload_that_fails_with_no_fresh_grant_is_reported_and_the_file_kept(home): + failing = AsyncMock(side_effect=TransferError("grant_expired", "expired")) + with patch.object(recording_module, "NamespaceUploader") as uploader: + uploader.return_value.upload = failing + _, manifest = await _record(Recorded(), 0.4, start={"upload": _grant()}) + (segment,) = manifest["segments"] + assert segment["uploaded"] is False and "grant_expired" in segment["upload_error"] + assert (home / ".agentenv-recordings" / segment["path"]).exists() + + +@pytest.mark.asyncio +async def test_retries_join_the_first_start_and_answer_end_again(home): + async with http_client(Recorded()) as http: + first = (await http.post(START, json={"request_id": "same"})).json() + again = (await http.post(START, json={"request_id": "same", "fps": 5})).json() + await asyncio.sleep(0.3) + ended = (await http.post(END, json={"recording_id": first["recording_id"]})).json() + repeat = (await http.post(END, json={"recording_id": first["recording_id"], "reason": "teardown"})).json() + assert again == first and repeat == ended and ended["ended_by"] == "end" + + +@pytest.mark.asyncio +async def test_a_start_past_max_concurrent_is_refused_and_a_reason_is_kept(home): + driver = FakeDriver() + driver.recording = Recording(max_concurrent=1) + async with http_client(driver) as http: + first = (await http.post(START, json={"request_id": "a"})).json() + refused = await http.post(START, json={"request_id": "b"}) + ended = (await http.post(END, json={"recording_id": first["recording_id"], "reason": "reset"})).json() + after = await http.post(START, json={"request_id": "c"}) + await http.post(END, json={"recording_id": after.json()["recording_id"]}) + assert refused.status_code == 429 and refused.json()["error"]["code"] == "limit_exceeded" + assert ended["ended_by"] == "reset" and after.status_code == 200 + + +@pytest.mark.asyncio +async def test_max_duration_and_a_full_disk_end_a_recording_as_limit(home, monkeypatch): + driver = FakeDriver() + driver.recording = Recording(max_duration_s=1) + _, timed = await _record(driver, 1.6) + monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 1 << 62) + _, full = await _record(Recorded(), 0.4, start={"request_id": "full"}) + assert timed["ended_by"] == "limit" and (_at(timed["ended_at"]) - _at(timed["started_at"])).total_seconds() < 1.2 + assert full["ended_by"] == "limit" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path, body, status, code", [ + (START, {"request_id": "r", "fps": 0}, 400, "invalid_argument"), + (START, {"request_id": "r", "fps": 30}, 400, "invalid_argument"), + (START, {"request_id": "r", "max_edge": 10}, 400, "invalid_argument"), + (START, {"request_id": "r", "scope": "region"}, 400, "invalid_argument"), + (START, {"request_id": "r", "scope": "window", "window": "w1"}, 501, "unsupported"), + (START, {"request_id": "r", "upload": {"root_path": "x"}}, 400, "invalid_argument"), + (START, {"request_id": ""}, 400, "invalid_argument"), + (END, {"recording_id": "rec-nope"}, 400, "invalid_argument"), + (END, {"recording_id": "rec-nope", "reason": "later"}, 400, "invalid_argument"), +]) +async def test_bad_calls_are_refused_with_a_code(path, body, status, code): + async with http_client(Recorded()) as http: + refused = await http.post(path, json=body) + assert refused.status_code == status and refused.json()["error"]["code"] == code + + +@pytest.mark.asyncio +async def test_without_pyav_a_start_is_refused_naming_the_extra(monkeypatch): + monkeypatch.setitem(__import__("sys").modules, "av", None) + async with http_client(Recorded()) as http: + refused = await http.post(START, json={"request_id": "r"}) + assert refused.status_code == 501 and "agentenv-framework-protocol[computer,recording]" in refused.text + + +@pytest.mark.asyncio +async def test_uploaded_segments_are_not_sent_again_at_end(home): + upload = AsyncMock() + with patch.object(recording_module, "NamespaceUploader") as uploader: + uploader.return_value.upload = upload + _, manifest = await _record(Recorded(), 1.4, start={"upload": _grant()}) + assert all(s["uploaded"] for s in manifest["segments"]) and upload.await_count == len(manifest["segments"]) >= 2 + + +@pytest.mark.asyncio +async def test_an_encoder_that_fails_on_its_first_frame_leaves_no_empty_segment(home, monkeypatch): + real, failed = recording_module._Encoder.add, [] + + def add(self, frame, at): + if not failed: + failed.append(at) + raise RuntimeError("encoder failed") + return real(self, frame, at) + + monkeypatch.setattr(recording_module._Encoder, "add", add) + _, manifest = await _record(Recorded(), 1.6) + assert failed and manifest["segments"] and all(s["frames"] > 0 for s in manifest["segments"]) + assert sorted(p.name for p in (home / ".agentenv-recordings" / manifest["recording_id"]).iterdir()) == [ + s["path"].rsplit("/", 1)[1] for s in manifest["segments"]] + + +@pytest.mark.asyncio +async def test_only_the_latest_starts_and_ends_are_kept_for_retries(home, monkeypatch): + monkeypatch.setattr(recording_module, "KEEP_ENDED", 1) + driver = Recorded() + async with http_client(driver) as http: + for name in "abcde": + started = (await http.post(START, json={"request_id": name})).json() + await http.post(END, json={"recording_id": started["recording_id"]}) + kept = driver._computer.recordings + assert list(kept.started) == list("bcde") and len(kept.ended) == 1 diff --git a/uv.lock b/uv.lock index 1fcfd34bb..cbd481459 100644 --- a/uv.lock +++ b/uv.lock @@ -211,6 +211,8 @@ computer = [ ] dev = [ { name = "a2a-sdk", extra = ["http-server"] }, + { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "fastmcp" }, { name = "mcp" }, { name = "pillow" }, @@ -219,11 +221,17 @@ dev = [ { name = "python-multipart" }, { name = "regex" }, ] +recording = [ + { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] [package.metadata] requires-dist = [ { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'agent'", specifier = ">=0.3.26,<0.4" }, { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'dev'", specifier = ">=0.3.26,<0.4" }, + { name = "av", marker = "extra == 'dev'", specifier = ">=17" }, + { name = "av", marker = "extra == 'recording'", specifier = ">=17" }, { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, @@ -242,7 +250,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "computer", "dev"] +provides-extras = ["agent", "computer", "recording", "dev"] [[package]] name = "aiofile" @@ -568,6 +576,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/c6/6f124bcfbbfb20fba22c939b4e43a06dccfc0e1ca20e5634ca573cb1e271/authlib-1.8.0-py2.py3-none-any.whl", hash = "sha256:88aebbd9af6757e14e912d5dc007ae1dc1f3e27e3b2152ce7c552ee2c3b3c121", size = 260804, upload-time = "2026-08-30T12:12:33.162Z" }, ] +[[package]] +name = "av" +version = "18.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/8d/f4/f22114d30d3435e38c6af2b4870f37b864403dca6ae7af747a289ce0a18e/av-18.1.0.tar.gz", hash = "sha256:47bfc286e1bc9de7ab4681fc2b575cd2460a66919d31ffe1bd5aa54fae531a28", size = 4451061, upload-time = "2026-08-12T22:28:18.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/d4/d7cdc8bff143c17a6d35924375ae28dd692cacde38700a7d419fde54f44a/av-18.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ae75d8bb6467895ed1f8572ededf7ffa49eac07f6e483222f5d7d62a41d12f04", size = 22546147, upload-time = "2026-08-12T22:27:11.851Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c9/37a619297492256b77d5ed906e7d8166c10a26ed251dccf1ae03ab19bff6/av-18.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:b30a4e8d934558e19602b68998a4d9ac9f250fa0dacef216f7e8e40153b13316", size = 18217603, upload-time = "2026-08-12T22:27:14.713Z" }, + { url = "https://files.pythonhosted.org/packages/d9/84/2464ffb64c08c5ce8b522c8e74594714414e3b0575267652c5c51c0574b9/av-18.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6fc837cc51adf80331ac850779cd53b5d4c4460b0ebe9057a02a921c6736f19d", size = 33640142, upload-time = "2026-08-12T22:27:17.835Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/204dbfc3e08eb4cdc6e6ff57be02150bc44523ebdb50182d10025792ebd9/av-18.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8a032e8d8ebc73dec079364b9b4a6837638a2d106e8472314e685ffbf163e700", size = 35786210, upload-time = "2026-08-12T22:27:20.984Z" }, + { url = "https://files.pythonhosted.org/packages/e1/99/b0d04ec553ff9a7e00455458dfa3a39c8a8f627b273056b4e5fe57d590de/av-18.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:3c8b1f8b46f99d52e2d8b0ed5d0cdadf172d24794d46e2077b16e44ed08e26ff", size = 39379798, upload-time = "2026-08-12T22:27:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/56/b1/e00d4feae59160149df6126585e726fdc6300798fd40c5dd324879e81f68/av-18.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ab5ac081bc9eaf54109120d4e56284674fecfbe520d9aa1707c7fa911ec5f4d2", size = 34690321, upload-time = "2026-08-12T22:27:27.769Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/836fa987e3084d11a21489f11357fb24843ef3aa8faf74ddddfc603d5062/av-18.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:191224788d87af06c31784a395bb73f14b72f33d7f4871ace0157de2abdc6276", size = 36859932, upload-time = "2026-08-12T22:27:31.403Z" }, + { url = "https://files.pythonhosted.org/packages/33/b4/76ba21e46704f632004276b85289a1582e95f5eff760436d6149875a1881/av-18.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:ea1480b7a8d5405cb5f382b344731bf125fd2c1c6fae3964f6c48595628387ff", size = 27595679, upload-time = "2026-08-12T22:27:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ad/a3135884c5753b09773176b97201ae602f67ad14206c395ff838d66bf9b0/av-18.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:5509ec12aaa19fd6601de13cfa6f4cdad450da07982118510592875d970454d6", size = 20257584, upload-time = "2026-08-12T22:27:38.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/5b/4a756265d7fb164336c8d377bca21c39cfa2c178be23cedee840a69b59c5/av-18.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:b36b0bae9e4c62f9487c99481ec15e4e3870fcc868522cd6d18fc2d6bfa04f01", size = 22795654, upload-time = "2026-08-12T22:27:42.016Z" }, + { url = "https://files.pythonhosted.org/packages/d5/cc/1bc841462114a1adf4f7d87456ab78a6972e23271e71865fcd2bbd0e7360/av-18.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:025f84494cb23278498f03b0d8117d3e47a1cbc9c44b97eb31875cf02251e46b", size = 18435735, upload-time = "2026-08-12T22:27:45.787Z" }, + { url = "https://files.pythonhosted.org/packages/b8/20/005500ed17a2e62a5e4bb94aa3786942560ec2f55ec1895ebf174c87abef/av-18.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:08a9ae288299cfcbf739dba4ad0c53b9b71f45184303dd45947920d022fed695", size = 37090807, upload-time = "2026-08-12T22:27:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f7/11e7f6d848d3690c31ca4f8578167393e619177f1493ccc93b9400852d4e/av-18.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cf8a17466bef07765dbdecc9e66ed9b25d20b4e14f654fbf35345a58ac45fa0c", size = 38976836, upload-time = "2026-08-12T22:27:54.565Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/b271473b24e806062d31191e40c6d65545e9cf59f80f044eba56dcbba0f4/av-18.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d49a5c542dfdc00f43c6cdb6cc41dac1781ee206fe180b56aa7433dfa816dfae", size = 40896630, upload-time = "2026-08-12T22:27:59.118Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9f/2ab7fa292a947ad3466ed8e655eefa3b82f535d7ea598c297b4471a937c4/av-18.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5548b79e2bf1f59b3e9aedc918a72d9dc45b9adaac10ff9470d5dbdda0002e47", size = 37895673, upload-time = "2026-08-12T22:28:03.98Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/04507c57249b399c3e4f23f01d221532f357338b5316fd2858fbd343127d/av-18.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7ea063f6690193ea335a1d592d6e0274350d45e2ed6af83ee107cb90cbfd84f", size = 39992431, upload-time = "2026-08-12T22:28:08.736Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d6/bc4b95bea9c2353a7e4d62a3fcfad9adcf0f881741c6ce01ee179d539ce3/av-18.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e4d48b9f12cad009cc72fe4f4099107de5e819c95f82767f4fd01a01481c0661", size = 28497798, upload-time = "2026-08-12T22:28:13.003Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d2/0c277a46f12647c1833f40496e132fb6001e0d19e6144b5ea30896461feb/av-18.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5cd9085028902c9880622bd37a12fd4b33060f06a52311f6f4867ca9f29a2c3b", size = 21421979, upload-time = "2026-08-12T22:28:16.48Z" }, +] + +[[package]] +name = "av" +version = "19.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'emscripten'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/23/01/76b6bf563fe907597cf58f10d10168e93fe588b6db9f2658db87e87efa7e/av-19.0.2.tar.gz", hash = "sha256:bebbc52598886480f4d861c575dc2b020785d6c04e1308680ddcc545e1b94c66", size = 4277157, upload-time = "2026-10-21T03:45:30.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/2f/f51f37820741134516e841053d5ecebf6350fbd451adcbff9ead96aa1244/av-19.0.2-cp312-abi3-macosx_11_0_x86_64.whl", hash = "sha256:f5f7bcf987567cd3f9b01f1c375c59bf59d8a5b98c9c5c154aeda7a11e0c5dce", size = 22627374, upload-time = "2026-10-21T03:44:28.296Z" }, + { url = "https://files.pythonhosted.org/packages/7c/42/067d0bcd2c07103f18cc23b510a8289dd80330e984c49d51ce0bafcee017/av-19.0.2-cp312-abi3-macosx_14_0_arm64.whl", hash = "sha256:b0dff6dfcc6ff9944fbbff15bc4ee2d90a2c1695d493bdea96e86f48823e9feb", size = 18556422, upload-time = "2026-10-21T03:44:31.267Z" }, + { url = "https://files.pythonhosted.org/packages/de/0c/448f89f76cf5e4da35dd6294a1e8155b75daa8839f2f2fa353775855d152/av-19.0.2-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:04c8e412642c652cb2b1997a9cb2114546ee7d894600c631f4be8ceb26a0d0d2", size = 32683287, upload-time = "2026-10-21T03:44:34.588Z" }, + { url = "https://files.pythonhosted.org/packages/e1/7f/9f1237feb894afa472ef834aa1fbfe4d6398f28dc7e86a1b418650204c36/av-19.0.2-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c1da7aad0fa6c13b4a5c1e7985be0d13f8526a96284573acaf3d55ead6d3fe8", size = 34990159, upload-time = "2026-10-21T03:44:37.637Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b5/1bde0e54e525d88ad8f5becae657f8142543fd3be9d10b72141f041ca868/av-19.0.2-cp312-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9d4f9f78c0ce1f20deb72626fdb3c723757f61a8cd6fbb7dd44d9401884a91fd", size = 44422922, upload-time = "2026-10-21T03:44:41.847Z" }, + { url = "https://files.pythonhosted.org/packages/08/21/65997929fa4dd7c20f64e742aa87205a8d65485718a2ab51633756078e83/av-19.0.2-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dfe7af5aa8243afd5d4778f85ad1a7c948e37fad272869da1f8f93f8d57fa7ad", size = 33754699, upload-time = "2026-10-21T03:44:45.772Z" }, + { url = "https://files.pythonhosted.org/packages/30/44/4de8c0ff5a5e25ceb0841efc0a4a7530a22d61800bc0cc19a1d03aa04358/av-19.0.2-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:44d3faae747cf04a33f39db8ae983ff3d93adc864ee663bc186b25b686803c3e", size = 36014845, upload-time = "2026-10-21T03:44:49.64Z" }, + { url = "https://files.pythonhosted.org/packages/32/5a/743a83666ded197df8b9957e2800bef9a39d52b23be2ca211bbd14b5fb22/av-19.0.2-cp312-abi3-win_amd64.whl", hash = "sha256:3ce5a1d7b916d13c16ac58f1756f0fee4af2d53f02bc1403e18550b50d375a22", size = 28150949, upload-time = "2026-10-21T03:44:53.434Z" }, + { url = "https://files.pythonhosted.org/packages/42/fe/e79d786628d9c0428f72214dbe4b90ee0c625e04d552a604d5235ff35515/av-19.0.2-cp312-abi3-win_arm64.whl", hash = "sha256:79a277b077e3768a0ea9f881ca0b3bc96d4135a8d3a7a579938adb58718d0dfd", size = 20708023, upload-time = "2026-10-21T03:44:56.435Z" }, + { url = "https://files.pythonhosted.org/packages/08/f5/e7988f64d3debf50748f1c4372a3dac70cfe53720563056cb57bd77771a6/av-19.0.2-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:b632e026f4375037438a6b2b5d6a489e55bc9504fad22078991ef96211bec2a9", size = 22911715, upload-time = "2026-10-21T03:44:59.378Z" }, + { url = "https://files.pythonhosted.org/packages/76/86/dbe91321e6b233970c7fdfdb82cb1750a745e5779ab9ee69a6ce2589b1cb/av-19.0.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:32a8d37bde9e24ad0cc855140cbeecde17a363d7dc7928dcfc71a34b5f3ed387", size = 18864540, upload-time = "2026-10-21T03:45:02.227Z" }, + { url = "https://files.pythonhosted.org/packages/5a/c7/afca86772d825f5eb5fd8d1a193b206ed9a4a0d74ffbe5bbe0a19521ed7d/av-19.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:1e7cb7b66a832bce8996af5b94f1cf4014a05ff51e0005feac4cb2566d711020", size = 36479296, upload-time = "2026-10-21T03:45:05.955Z" }, + { url = "https://files.pythonhosted.org/packages/b3/71/6c6fc7e80dd5aa71109cb9324638620c40b7de8f677a6446d2f744295587/av-19.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b559ff381bd800c46fee6a17d7e22a3443708cdd58d0901ae91d27cf5dd0578c", size = 38467551, upload-time = "2026-10-21T03:45:09.493Z" }, + { url = "https://files.pythonhosted.org/packages/3f/93/fa67f5d4180e57e7f740053fd855a6d312f4848f1e5582b8d3ab131184dd/av-19.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:91924f5c836b1cbf89fe1522c642f98e89a733124908561fd49dc7a70e9b6d10", size = 46223361, upload-time = "2026-10-21T03:45:13.396Z" }, + { url = "https://files.pythonhosted.org/packages/25/f9/f98627770ccb4761e8cb87edd8b4bcf91549288bca95130c1a91ea7cdcb8/av-19.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ae4618eab172f8135596972252f39c7399291c921e661c1771ba26b9254b800f", size = 37349208, upload-time = "2026-10-21T03:45:17.213Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3a/183b4f96a604a8544180e256d16f9a6c0534d6b7db22dc22fb0842aa96fc/av-19.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8596fad2017d380708de35ddf9d34f808b8488c912cdc3f361691d866d9dc32d", size = 39475989, upload-time = "2026-10-21T03:45:20.658Z" }, + { url = "https://files.pythonhosted.org/packages/22/6e/84b65c58171a6d75a5d129a9cf856ca730843f8c476f08033e7f9d4a1af6/av-19.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9db27c2248112a8eed3edd3ca83cf4405f52a402cda2eb1956035e646e6b99c6", size = 29064804, upload-time = "2026-10-21T03:45:24.24Z" }, + { url = "https://files.pythonhosted.org/packages/60/4e/d96cc9fff457069c52754f63151a0340cb4886330d58e09061f203696b60/av-19.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:0e425d631510ca21e47c58f3837a9dceae0453c24e926408c8a12b95239cbbd6", size = 21916888, upload-time = "2026-10-21T03:45:27.298Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" From d333a7d010e4ba2ea2bde8ec66a9b43c9f53d714 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:28:44 -0700 Subject: [PATCH 02/17] feat(protocol): probe a driver's recordings At full, a driver that declares computer-recording/v1 records without a grant for two seconds: the probe reads each segment back through computer-exec and checks its size, SHA-256 and MP4 header (size, frame count, duration) against the manifest, with no gap between segments; a repeated request_id joins the first recording; a recording past max_concurrent is refused limit_exceeded. Every recording the probe made is ended and removed. The MP4 reader parses moov's tkhd, mdhd and stts, with no decoder. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 91 +++++++++- .../computer/conformance/mp4.py | 65 +++++++ .../test_computer_conformance_recording.py | 169 ++++++++++++++++++ 3 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/mp4.py create mode 100644 packages/agentenv-protocol/tests/test_computer_conformance_recording.py diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index f5c007d9a..5569cb78a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -4,11 +4,14 @@ from __future__ import annotations import asyncio +import base64 +import contextlib import json import math import time import uuid from collections.abc import Awaitable, Callable +from datetime import datetime from importlib import resources from mcp import ClientSession @@ -25,11 +28,14 @@ COMPUTER_KEYBOARD, COMPUTER_POINTER, COMPUTER_READY, + COMPUTER_RECORDING, COMPUTER_SPACES, GROUP_TOOLS, POINT_TOOLS, ) +from ..recording import RECORDINGS_DIR from .client import Driver, image +from .mp4 import read_mp4 from .report import CheckResult MINIMUM_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", @@ -45,6 +51,12 @@ #: How long the fixture may take to start. FIXTURE_START_S = 30.0 NORM = 1000 +#: How long the probe records, and the frame rate it asks for. +RECORD_S = 2.0 +RECORD_FPS = 5 +#: Prints a file's size, SHA-256 and bytes as JSON, run with ``python3 -c`` on the computer. +READ_FILE = ("import base64,hashlib,json,sys;d=open(sys.argv[1],'rb').read();" + "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest(),'data':base64.b64encode(d).decode()}))") #: The checks only ``full`` runs, recorded unverified when the fixture can't be staged. FULL_CHECKS = (("P3", "fixture colours land where the mapping says"), ("P4", "clicks show their effect"), ("P5", "refusals leave the fixture as it was"), ("P6", "typed and pressed keys arrive"), @@ -89,7 +101,7 @@ async def full(self) -> None: return self._skip_full(f"the fixture didn't start: {problem}") for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), - ("X", self._pointer_input), ("X", self._data_add)): + ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording)): await self._guarded(check, section) finally: await self._unstage() @@ -721,6 +733,83 @@ async def picked_up() -> str | None: await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE) + async def _recording(self) -> None: + if not self.driver.declares(COMPUTER_RECORDING): + return + params = self.driver.params(COMPUTER_RECORDING) + fps = min(RECORD_FPS, (params.get("fps") or {}).get("max") or RECORD_FPS) + made: list[str] = [] + state: dict = {} + + async def start(request_id: str) -> dict: + started = await self.driver.invoke(COMPUTER_RECORDING, "start", {"request_id": request_id, "fps": fps}) + if started["recording_id"] not in made: + made.append(started["recording_id"]) + return started + + async def recorded() -> str | None: + request_id = f"{PROBE_ROLE}-recording-{uuid.uuid4().hex[:8]}" + started = await start(request_id) + state["joined"] = await start(request_id) == started + await asyncio.sleep(RECORD_S) + manifest = await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": started["recording_id"]}) + return await self._segments_match(manifest, fps) + + async def joined() -> str | None: + return None if state.get("joined") else "a repeated request_id started a second recording" + + async def limited() -> str | None: + most = params.get("max_concurrent") + if not isinstance(most, int): + return "the card declares no max_concurrent" + for _ in range(most): + await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + try: + extra = await start(f"{PROBE_ROLE}-limit-{uuid.uuid4().hex[:8]}") + except ExtensionCallError as e: + return None if e.code == "limit_exceeded" else f"refused {e.code!r}, not 'limit_exceeded': {e.detail}" + return f"recording {most + 1} of at most {most} started: {extra['recording_id']}" + + try: + await self.check("X", "a recording's segments match their files, with no gap between them", recorded, + COMPUTER_RECORDING) + await self.check("X", "a repeated request_id joins the first recording", joined, COMPUTER_RECORDING) + await self.check("X", "a recording past max_concurrent is refused", limited, COMPUTER_RECORDING) + finally: + for recording_id in made: + with contextlib.suppress(Exception): + await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) + await self.driver.run("rm", "-rf", f"{RECORDINGS_DIR}/{recording_id}") + + async def _segments_match(self, manifest: dict, fps: int) -> str | None: + """What is wrong with a recording without a grant, read back from the computer, or None.""" + segments = manifest.get("segments") or [] + if manifest.get("ended_by") != "end" or not segments: + return f"it ended by {manifest.get('ended_by')!r} with {len(segments)} segments" + for before, after in zip(segments, segments[1:]): + gap = (_when(after["started_at"]) - _when(before["ended_at"])).total_seconds() + if gap > 2 / fps: + return f"{gap:.2f} s between segments {before['index']} and {after['index']}" + for segment in segments: + out = await self.driver.run("python3", "-c", READ_FILE, f"{RECORDINGS_DIR}/{segment['path']}") + if out.get("exit_code") != 0 or out.get("truncated"): + return f"segment {segment['index']} couldn't be read back: {out.get('stderr', '')[-200:]}" + file = json.loads(out["stdout"]) + if (file["size"], file["sha256"]) != (segment["bytes"], segment["sha256"]): + return f"segment {segment['index']}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ + f"the manifest says {segment['bytes']}, {segment['sha256'][:12]}" + mp4 = read_mp4(base64.b64decode(file["data"])) + span = (_when(segment["ended_at"]) - _when(segment["started_at"])).total_seconds() + if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment["width"], segment["height"], segment["frames"]) + or abs(mp4["duration_s"] - span) > 2 / fps): + return f"segment {segment['index']}'s MP4 is {mp4}; the manifest says {segment['width']}x" \ + f"{segment['height']}, {segment['frames']} frames over {span:.2f} s" + return None + + +def _when(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + def _rgb(colour: str) -> tuple[int, int, int]: return int(colour[1:3], 16), int(colour[3:5], 16), int(colour[5:7], 16) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/mp4.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/mp4.py new file mode 100644 index 000000000..e8607ef37 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/mp4.py @@ -0,0 +1,65 @@ +"""What the probe reads from an MP4 without decoding it: the video track's size, frame count and duration, from the +``moov`` box's ``tkhd``, ``mdhd`` and ``stts``.""" +from __future__ import annotations + +import struct +from collections.abc import Iterator + + +def read_mp4(data: bytes) -> dict: + """``{"width", "height", "frames", "duration_s"}`` of the first video track. Raises ValueError for a file that + isn't an MP4 with one.""" + top = dict(_boxes(data, 0, len(data))) + if b"ftyp" not in top or b"moov" not in top: + raise ValueError(f"not an MP4: its top-level boxes are {sorted(t.decode(errors='replace') for t in top)}") + for trak in _children(data, top[b"moov"], b"trak"): + mdia = _child(data, trak, b"mdia") + hdlr = _child(data, mdia, b"hdlr") + if data[hdlr[0] + 8:hdlr[0] + 12] != b"vide": + continue + width, height = _tkhd_size(data, _child(data, trak, b"tkhd")) + timescale, duration = _mdhd(data, _child(data, mdia, b"mdhd")) + stts = _child(data, _child(data, _child(data, mdia, b"minf"), b"stbl"), b"stts") + entries = struct.unpack(">I", data[stts[0] + 4:stts[0] + 8])[0] + frames = sum(struct.unpack(">I", data[stts[0] + 8 + 8 * i:stts[0] + 12 + 8 * i])[0] for i in range(entries)) + return {"width": width, "height": height, "frames": frames, "duration_s": duration / timescale} + raise ValueError("the MP4 has no video track") + + +def _boxes(data: bytes, start: int, end: int) -> Iterator[tuple[bytes, tuple[int, int]]]: + """Each box between ``start`` and ``end``: its type, and where its payload starts and ends.""" + at = start + while at + 8 <= end: + size, kind = struct.unpack(">I4s", data[at:at + 8]) + header = 8 + if size == 1: + size, header = struct.unpack(">Q", data[at + 8:at + 16])[0], 16 + elif size == 0: + size = end - at + if size < header or at + size > end: + raise ValueError(f"a {kind.decode(errors='replace')} box runs past its container") + yield kind, (at + header, at + size) + at += size + + +def _children(data: bytes, box: tuple[int, int], kind: bytes) -> list[tuple[int, int]]: + return [span for k, span in _boxes(data, *box) if k == kind] + + +def _child(data: bytes, box: tuple[int, int], kind: bytes) -> tuple[int, int]: + found = _children(data, box, kind) + if not found: + raise ValueError(f"no {kind.decode()} box where the MP4 needs one") + return found[0] + + +def _tkhd_size(data: bytes, box: tuple[int, int]) -> tuple[int, int]: + at = box[0] + (88 if data[box[0]] == 1 else 76) + width, height = struct.unpack(">II", data[at:at + 8]) + return width >> 16, height >> 16 + + +def _mdhd(data: bytes, box: tuple[int, int]) -> tuple[int, int]: + if data[box[0]] == 1: + return struct.unpack(">IQ", data[box[0] + 20:box[0] + 32]) + return struct.unpack(">II", data[box[0] + 12:box[0] + 20]) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py new file mode 100644 index 000000000..20ecdfef6 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -0,0 +1,169 @@ +"""The probe's recording checks against a served driver that records: they pass, and they fail on a manifest whose +hash or size isn't the file's, on a card that declares a limit the driver doesn't keep, and on an MP4 that isn't one; +every recording the probe made is ended and removed.""" +from __future__ import annotations + +import struct + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") +pytest.importorskip("av") + +from probe_fake import FixtureDriver, served # noqa: E402 + +from agentenv_protocol.computer import Recording, capture # noqa: E402 +from agentenv_protocol.computer import recording as recording_module # noqa: E402 +from agentenv_protocol.computer.conformance import checks, run_probe # noqa: E402 +from agentenv_protocol.computer.conformance.client import Driver # noqa: E402 +from agentenv_protocol.computer.conformance.mp4 import read_mp4 # noqa: E402 + +NAMES = ("a recording's segments match their files, with no gap between them", + "a repeated request_id joins the first recording", "a recording past max_concurrent is refused") + + +@pytest.fixture(autouse=True) +def home(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(checks, "RECORD_S", 0.8) + return tmp_path + + +class Recorder(FixtureDriver): + recording = Recording(max_concurrent=2) + + +async def _recording_checks(driver) -> dict[str, tuple[str, str]]: + with served(driver) as url: + connected = await Driver.connect(url, 10) + async with connected.session() as session: + probe = checks.Probe(connected, session, 10) + await probe._recording() + return {r.name: (r.status, r.detail) for r in probe.results} + + +@pytest.mark.asyncio +async def test_a_driver_that_records_passes_the_recording_checks_at_full_and_leaves_nothing(home): + with served(Recorder()) as url: + report = await run_probe(url, "full") + statuses = {c.name: c.status for c in report.checks} + assert report.passed and all(statuses[name] == "passed" for name in NAMES) + assert report.extensions["urn:agentenv:computer-recording/v1"] == "verified" + assert not any((home / ".agentenv-recordings").iterdir()) + + +@pytest.mark.asyncio +async def test_a_manifest_whose_hash_or_size_isnt_the_files_fails(home, monkeypatch): + real = recording_module._digest + monkeypatch.setattr(recording_module, "_digest", lambda path: (real(path)[0], "0" * 64)) + wrong_hash = await _recording_checks(Recorder()) + monkeypatch.setattr(recording_module, "_digest", lambda path: (real(path)[0] + 1, real(path)[1])) + wrong_size = await _recording_checks(Recorder()) + assert wrong_hash[NAMES[0]][0] == "failed" and "sha256 " in wrong_hash[NAMES[0]][1] + assert wrong_size[NAMES[0]][0] == "failed" and "bytes" in wrong_size[NAMES[0]][1] + + +@pytest.mark.asyncio +async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still_ended(home, monkeypatch): + advertised = recording_module.recording_params + monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", + lambda features, streams: {**advertised(features, streams), "max_concurrent": 1}) + driver = Recorder() + found = await _recording_checks(driver) + assert found[NAMES[2]][0] == "failed" and "recording 2 of at most 1 started" in found[NAMES[2]][1] + assert found[NAMES[0]][0] == found[NAMES[1]][0] == "passed" + assert not driver._computer.recordings.running and not any((home / ".agentenv-recordings").iterdir()) + + +@pytest.mark.asyncio +async def test_a_driver_without_recording_gets_no_recording_checks(home): + assert await _recording_checks(FixtureDriver()) == {} + + +def _box(kind: bytes, payload: bytes) -> bytes: + return struct.pack(">I4s", 8 + len(payload), kind) + payload + + +def test_the_mp4_reader_refuses_what_isnt_an_mp4_with_a_video_track(): + with pytest.raises(ValueError, match="not an MP4"): + read_mp4(_box(b"free", b"")) + hdlr = _box(b"hdlr", b"\0" * 8 + b"soun" + b"\0" * 12) + audio = _box(b"ftyp", b"isom") + _box(b"moov", _box(b"trak", _box(b"mdia", hdlr))) + with pytest.raises(ValueError, match="no video track"): + read_mp4(audio) + with pytest.raises(ValueError, match="runs past its container"): + read_mp4(_box(b"ftyp", b"isom") + struct.pack(">I4s", 400, b"moov")) + vide = _box(b"hdlr", b"\0" * 8 + b"vide" + b"\0" * 12) + no_tkhd = _box(b"ftyp", b"isom") + _box(b"moov", _box(b"trak", _box(b"mdia", vide))) + with pytest.raises(ValueError, match="no tkhd box"): + read_mp4(no_tkhd) + + +def _mp4(*, large: bool, version: int) -> bytes: + tkhd = b"\0" * (88 if version else 76) + struct.pack(">II", 320 << 16, 200 << 16) + tkhd = bytes([version]) + tkhd[1:] + mdhd = (bytes([1]) + b"\0" * 19 + struct.pack(">IQ", 1000, 2500)) if version else ( + b"\0" * 12 + struct.pack(">II", 1000, 2500)) + stts = b"\0" * 4 + struct.pack(">III", 1, 25, 100) + vide = _box(b"hdlr", b"\0" * 8 + b"vide" + b"\0" * 12) + mdia = _box(b"mdia", vide + _box(b"mdhd", mdhd) + _box(b"minf", _box(b"stbl", _box(b"stts", stts)))) + moov = _box(b"moov", _box(b"trak", _box(b"tkhd", tkhd) + mdia)) + if large: + moov = struct.pack(">I4sQ", 1, b"moov", 16 + len(moov) - 8) + moov[8:] + return _box(b"ftyp", b"isom") + moov + struct.pack(">I4s", 0, b"mdat") + b"frames" + + +@pytest.mark.parametrize("large, version", [(False, 0), (True, 1)]) +def test_the_mp4_reader_takes_large_boxes_a_box_to_the_end_and_version_1_headers(large, version): + assert read_mp4(_mp4(large=large, version=version)) == {"width": 320, "height": 200, "frames": 25, + "duration_s": 2.5} + + +class Gappy(Recorder): + """Its captures fail once, a little into a recording.""" + calls = 0 + + @capture + async def grab(self): + Gappy.calls += 1 + if Gappy.calls == 3: + raise RuntimeError("the X server went away") + return await super().grab() + + +@pytest.mark.asyncio +async def test_each_way_a_recording_can_fall_short_fails_its_check(home, monkeypatch): + monkeypatch.setattr(checks, "RECORD_S", 2.2) + gap = await _recording_checks(Gappy()) + monkeypatch.setattr(checks, "RECORD_S", 0.8) + monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 1 << 62) + limited = await _recording_checks(Recorder()) + monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 0) + monkeypatch.setattr(checks, "READ_FILE", "import sys; sys.exit(3)") + unreadable = await _recording_checks(Recorder()) + monkeypatch.undo() + monkeypatch.setattr(checks, "RECORD_S", 0.8) + real_end = recording_module.Recordings.end + + async def miscounted(self, recording_id, upload, reason): + manifest = await real_end(self, recording_id, upload, reason) + manifest["segments"][0]["frames"] += 1 + return manifest + + monkeypatch.setattr(recording_module.Recordings, "end", miscounted) + miscount = await _recording_checks(Recorder()) + assert "s between segments 0 and 1" in gap[NAMES[0]][1] + assert "it ended by 'limit'" in limited[NAMES[0]][1] + assert "couldn't be read back" in unreadable[NAMES[0]][1] + assert "the manifest says 1280x800" in miscount[NAMES[0]][1] + + +@pytest.mark.asyncio +async def test_a_card_with_no_max_concurrent_fails_the_limit_check(home, monkeypatch): + advertised = recording_module.recording_params + monkeypatch.setattr("agentenv_protocol.computer.card.recording_params", + lambda features, streams: {k: v for k, v in advertised(features, streams).items() + if k != "max_concurrent"}) + found = await _recording_checks(Recorder()) + assert found[NAMES[2]] == ("failed", "the card declares no max_concurrent") From 58f15dab21dc58b0d35cb7b472abc42a5f24d444 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:36:08 -0700 Subject: [PATCH 03/17] feat(task): record a computer env with start_computer_recording and end_computer_recording start_computer_recording starts a recording on a deployed env whose card declares computer-recording/v1, with a write_namespace grant under agentenv-recordings/ that lasts the env's max_duration_s and a margin, or the longest the store signs. end_computer_recording ends it with a fresh grant on the same namespace and registers the uploaded segments and the manifest as FileArtifacts, pointing at the objects, in one FileArtifactUniverse; context.metadata["computer_recordings"][name] says how it went. A recording its driver no longer knows, as after a restart, is registered from what its namespace holds, ended_by error. teardown_run, the teardown_sandboxes step and reset_env end the recordings a run left open first, with reason teardown or reset. Both steps are best effort unless fail_task_on_error. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task/teardown.py | 12 +- src/agent_env/task_step/registry.py | 6 + .../task_steps/computer_recording.py | 210 ++++++++++++++++++ .../task_step/task_steps/reset_env.py | 2 + .../task_steps/teardown_sandboxes.py | 3 + tst/unit/task_step/computer_recording_test.py | 202 +++++++++++++++++ 7 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/computer_recording.py create mode 100644 tst/unit/task_step/computer_recording_test.py diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 5090f6efe..1719ac289 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -148,7 +148,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s UNAFFECTED = frozenset({ # They name no agent. "aggregate_verifiers", "apply_server_config", "build_mcp_cli", "combine_universe_verdicts", "deploy_env", - "deploy_sandbox", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", + "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", "verify_spec_conformance", "verify_universe_load_export_roundtrip", diff --git a/src/agent_env/task/teardown.py b/src/agent_env/task/teardown.py index abe2372cc..b5fe16062 100644 --- a/src/agent_env/task/teardown.py +++ b/src/agent_env/task/teardown.py @@ -2,8 +2,8 @@ ``teardown_run`` terminates every sandbox a run's context records: its bare sandboxes', its envs' and its agents'. It removes compute only; the instance, its outputs and the artifacts it collected stay. A local -sandbox's work folder goes too, once the sandbox is down. ``agent-env run`` and ``agent-env eval run`` call -it after each run. ``Task.run`` never does, so a caller that keeps a run's environment up, as the hub and +sandbox's work folder goes too, once the sandbox is down. Recordings still open are ended first. ``agent-env run`` and ``agent-env +eval run`` call it after each run. ``Task.run`` never does, so a caller that keeps a run's environment up, as the hub and the worker do, owns its teardown. """ @@ -20,6 +20,7 @@ from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider from agent_env.task.interrupts import tearing_down from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.computer_recording import end_open_recordings from agent_env.task_step.task_steps.teardown_sandboxes import ( _DEFAULT_PROVIDERS, TORN_DOWN_KEY, _env_sandbox_ids, _sandbox_ids_of, ) @@ -104,6 +105,13 @@ async def teardown_run(context: TaskStepContext, *, timeout: float = TERMINATE_T logger.warning("teardown: staged changelog increments were not all drained within %gs", timeout) except asyncio.CancelledError: return TeardownReport(left=pending) + try: # a recording still open is ended, its last segment uploaded and registered, before its computer goes + async with asyncio.timeout(timeout): + await end_open_recordings(context, "teardown") + except TimeoutError: + logger.warning("teardown: open recordings were not all ended within %gs", timeout) + except asyncio.CancelledError: + return TeardownReport(left=pending) async def take_down(sandbox: RecordedSandbox) -> None: if sandbox not in pending: # torn down already: only a local one's folder can be left diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 76d1934c0..d0c32afca 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -28,6 +28,10 @@ def get_task_step_registry() -> dict[str, type["TaskStep"]]: def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep + from agent_env.task_step.task_steps.computer_recording import ( + EndComputerRecordingTaskStep, + StartComputerRecordingTaskStep, + ) from agent_env.task_step.task_steps.mcp_cli_builder.build_mcp_cli import BuildMcpCliTaskStep from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep @@ -81,6 +85,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(ReviewTaskStep): ReviewTaskStep, _get_type(BuildMcpCliTaskStep): BuildMcpCliTaskStep, _get_type(CollectArtifactsTaskStep): CollectArtifactsTaskStep, + _get_type(StartComputerRecordingTaskStep): StartComputerRecordingTaskStep, + _get_type(EndComputerRecordingTaskStep): EndComputerRecordingTaskStep, _get_type(DeployAgentTaskStep): DeployAgentTaskStep, _get_type(DeployEnvTaskStep): DeployEnvTaskStep, _get_type(DeployHumanAgentTaskStep): DeployHumanAgentTaskStep, diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py new file mode 100644 index 000000000..0f155c888 --- /dev/null +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -0,0 +1,210 @@ +"""Record a computer env's display. ``start_computer_recording`` starts a recording on a deployed env whose card +declares computer-recording/v1, with a grant to upload its segments under a namespace of its own; +``end_computer_recording`` ends it and registers its segments and manifest as one FileArtifactUniverse. Teardown +and ``reset_env`` end whatever a run left open (``end_open_recordings``).""" +from __future__ import annotations + +import json +import logging +import uuid +from typing import ClassVar, Optional + +from agentenv_protocol.computer.card import COMPUTER_RECORDING +from agentenv_protocol.client import find_extension + +from agent_env.a2a_agent.object_transfer import ObjectLimits, namespace_grant +from agent_env.artifact import FileArtifact +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.artifact.store import get_artifact_store +from agent_env.config import get_config +from agent_env.entity.refs import EntityRef +from agent_env.store.base import GrantUnavailableError +from agent_env.store.ids import derive_id, is_local_id, validate_local_id +from agent_env.store.object_store.object_store import issues_grants_to +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency + +logger = logging.getLogger(__name__) + +#: ``context.metadata[RECORDINGS_KEY][name]``: each recording a run started, open or ended. +RECORDINGS_KEY = "computer_recordings" +_KEY_PREFIX = "agentenv-recordings" +#: How much longer than the env's ``max_duration_s`` a recording's grant lasts. +GRANT_MARGIN_S = 600 +_MAX_SEGMENT_BYTES = 5 << 30 + + +class StartComputerRecordingTaskStep(TaskStep): + """Start recording a deployed env's display, under ``name``. Best effort by default.""" + + type: ClassVar[str] = "start_computer_recording" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, name: str, fps: Optional[int] = None, + max_edge: Optional[int] = None, cursor: bool = True, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + if not env_id or not name: + raise ValueError("start_computer_recording needs an env_id and a name for the recording") + self.env_id, self.name, self.fps, self.max_edge, self.cursor = env_id, name, fps, max_edge, cursor + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "name": self.name, "fps": self.fps, + "max_edge": self.max_edge, "cursor": self.cursor} + + @classmethod + def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: + base = cls._base_from_dict(data) + base["fail_task_on_error"] = data.get("fail_task_on_error", False) + return cls(**base, env_id=data["env_id"], name=data["name"], fps=data.get("fps"), + max_edge=data.get("max_edge"), cursor=data.get("cursor", True)) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + try: + deployed = _deployed(context, self.env_id) + namespace, grant = _grant(deployed) + started = await deployed.invoke(COMPUTER_RECORDING, "start", { + "request_id": f"{context.instance_id}:{self.id}", "fps": self.fps, "max_edge": self.max_edge, + "cursor": self.cursor, "upload": grant}) + except Exception as e: + context.metadata.setdefault(RECORDINGS_KEY, {})[self.name] = { + "env_id": self.env_id, "state": "failed", "error": f"{type(e).__name__}: {e}"[:500]} + logger.warning("start_computer_recording %s on %s: %s", self.name, self.env_id, e) + if self.fail_task_on_error: + raise + return context + context.metadata.setdefault(RECORDINGS_KEY, {})[self.name] = { + "env_id": self.env_id, "recording_id": started["recording_id"], "started_at": started["started_at"], + "namespace": namespace, "state": "open"} + logger.info("start_computer_recording %s on %s: %s", self.name, self.env_id, started["recording_id"]) + return context + + +class EndComputerRecordingTaskStep(TaskStep): + """End the recording ``name`` and register it. Best effort by default.""" + + type: ClassVar[str] = "end_computer_recording" + entity_refs = () + + def __init__(self, id: str, version: Optional[int], name: str, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = False): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + if not name: + raise ValueError("end_computer_recording needs the name its start_computer_recording gave") + self.name = name + + def to_dict(self) -> dict: + return {**super().to_dict(), "name": self.name} + + @classmethod + def from_dict(cls, data: dict) -> EndComputerRecordingTaskStep: + base = cls._base_from_dict(data) + base["fail_task_on_error"] = data.get("fail_task_on_error", False) + return cls(**base, name=data["name"]) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + entry = (context.metadata.get(RECORDINGS_KEY) or {}).get(self.name) + if entry is None: + error = RuntimeError(f"no recording named {self.name!r} was started in this run") + logger.warning("end_computer_recording: %s", error) + if self.fail_task_on_error: + raise error + return context + await end_recording(context, self.name, "end") + if self.fail_task_on_error and entry["state"] != "ended": + raise RuntimeError(f"recording {self.name!r} didn't end: {entry.get('error')}") + return context + + +async def end_open_recordings(context: TaskStepContext, reason: str, env_ids: Optional[list[str]] = None) -> None: + """End and register every recording still open in ``context`` (of ``env_ids`` only, when given).""" + for name, entry in list((context.metadata.get(RECORDINGS_KEY) or {}).items()): + if entry.get("state") == "open" and (env_ids is None or entry["env_id"] in env_ids): + await end_recording(context, name, reason) + + +async def end_recording(context: TaskStepContext, name: str, reason: str) -> None: + """End the open recording ``name`` with a fresh grant and register it. When the driver no longer knows it, as + after a restart, what its namespace holds is registered instead. Never raises: the entry says what happened.""" + entry = context.metadata[RECORDINGS_KEY][name] + if entry.get("state") != "open": + return + try: + deployed = _deployed(context, entry["env_id"]) + _, grant = _grant(deployed, entry["namespace"]) + manifest = await deployed.invoke(COMPUTER_RECORDING, "end", { + "recording_id": entry["recording_id"], "upload": grant, "reason": reason}) + except Exception as e: + logger.warning("end_computer_recording %s: %s; registering what its namespace holds", name, e) + entry["error"] = f"{type(e).__name__}: {e}"[:500] + manifest = _from_namespace(entry) + try: + entry["artifact"] = _register(context, name, entry["namespace"], manifest) + except Exception as e: + logger.warning("end_computer_recording %s: registering it failed: %s", name, e) + entry["state"], entry["error"] = "failed", f"{type(e).__name__}: {e}"[:500] + return + uploaded = [s for s in manifest["segments"] if s.get("uploaded")] + entry.update(state="ended", ended_by=manifest["ended_by"], segments=len(uploaded), + missing_segments=len(manifest["segments"]) - len(uploaded)) + logger.info("end_computer_recording %s: %d segments, ended by %s", name, len(uploaded), manifest["ended_by"]) + + +def _deployed(context: TaskStepContext, env_id: str): + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"env {env_id!r} isn't deployed in this run") + return deployed + + +def _grant(deployed, namespace: Optional[str] = None) -> tuple[str, dict]: + """A namespace for the recording's segments (a new one unless ``namespace``) and a grant that uploads under it, + lasting the env's ``max_duration_s`` and a margin, or the longest the store issues.""" + params = (find_extension(deployed.environment_card or {}, COMPUTER_RECORDING) or {}).get("params") or {} + store, config = get_config().get_object_store(), get_config() + sandbox_type = getattr(deployed, "sandbox_type", None) + if not issues_grants_to(store, sandbox_type): + raise RuntimeError(f"the object store issues no upload grant that reaches env {deployed.env_id}'s sandbox " + f"({sandbox_type or 'unknown'}), so its recording couldn't upload") + if namespace is None: + namespace = store.object_url(f"{config.get_artifact_key_prefix()}{_KEY_PREFIX}/{uuid.uuid4().hex}") + duration, segment = params.get("max_duration_s", 3600), params.get("max_segment_s", 300) + cap = min(store.max_single_upload_bytes or _MAX_SEGMENT_BYTES, _MAX_SEGMENT_BYTES) + limits = ObjectLimits(max_objects=duration // segment + 2, max_object_bytes=cap, + max_total_bytes=cap * (duration // segment + 2)) + try: + grant = namespace_grant(store, namespace, limits=limits, expires_in=duration + GRANT_MARGIN_S) + except GrantUnavailableError: + grant = namespace_grant(store, namespace, limits=limits, expires_in=store.grant_lifetime_seconds) + return namespace, grant.model_dump(mode="json") + + +def _from_namespace(entry: dict) -> dict: + """A manifest of what the recording's namespace holds, for a recording its driver can't end.""" + store = get_config().get_object_store() + root = entry["namespace"].rstrip("/") + "/" + paths = sorted(url[len(root):] for url in store.list_at(root) if url.endswith(".mp4")) + return {"recording_id": entry.get("recording_id"), "ended_by": "error", "error": entry.get("error"), + "segments": [{"index": i, "path": path, "uploaded": True} for i, path in enumerate(paths)]} + + +def _register(context: TaskStepContext, name: str, namespace: str, manifest: dict) -> dict: + """The uploaded segments and the manifest as FileArtifacts, pointing at the objects, in one universe.""" + universe_id = derive_id(context.instance_id or "run", f"recording-{name}") + if is_local_id(universe_id): + validate_local_id(universe_id) + objects, documents = get_config().get_object_store(), get_artifact_store() + root = namespace.rstrip("/") + manifest_url = objects.put(objects.get_object_key(f"{root}/manifest.json"), + json.dumps(manifest, indent=1).encode(), content_type="application/json", + allow_overwrite=True) + files = {"manifest.json": (manifest_url, "application/json")} + for segment in manifest["segments"]: + if segment.get("uploaded"): + files[segment["path"].replace("/", "-")] = (f"{root}/{segment['path']}", "video/mp4") + stored = {filename: documents.put_document(FileArtifact( + id=derive_id(universe_id, filename), filename=filename, content_type=content_type, object_url=url, + description=f"Recording {name!r} of task instance {context.instance_id}")) + for filename, (url, content_type) in files.items()} + universe = FileArtifactUniverse.put(id=universe_id, file_artifacts=stored) + return {"id": universe.id, "version": universe.version} diff --git a/src/agent_env/task_step/task_steps/reset_env.py b/src/agent_env/task_step/task_steps/reset_env.py index 88059fdde..15fb14eb6 100644 --- a/src/agent_env/task_step/task_steps/reset_env.py +++ b/src/agent_env/task_step/task_steps/reset_env.py @@ -11,6 +11,7 @@ from typing import Optional from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.computer_recording import end_open_recordings from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep @@ -57,6 +58,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if deployed is None: logger.warning("reset_env: no deployed env '%s' in context; skipping", self.env_id) return context + await end_open_recordings(context, "reset", env_ids=[self.env_id]) try: env = Env.get(deployed.env_id, deployed.env_version) await env.reset(deployed) diff --git a/src/agent_env/task_step/task_steps/teardown_sandboxes.py b/src/agent_env/task_step/task_steps/teardown_sandboxes.py index de191ff31..e122e064a 100644 --- a/src/agent_env/task_step/task_steps/teardown_sandboxes.py +++ b/src/agent_env/task_step/task_steps/teardown_sandboxes.py @@ -25,6 +25,7 @@ get_sandbox_provider, ) from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.computer_recording import end_open_recordings from agent_env.task_step.task_step import TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -87,6 +88,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning(f"teardown_sandboxes: not deployed in this run, skipping: {', '.join(missing)}") already = set(context.metadata.get(TORN_DOWN_KEY) or []) pending = {sid: (stype, default) for sid, (stype, default) in targets.items() if sid not in already} + if self.env_ids: + await end_open_recordings(context, "teardown", env_ids=self.env_ids) for name in self.agent_names: # a changelog staged on an agent goes into the store before the agent does await drain_all(staged_changelogs(context.metadata, agent_name=name), within=LAST_DRAIN_SECONDS) results = await asyncio.gather( diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py new file mode 100644 index 000000000..6b8200749 --- /dev/null +++ b/tst/unit/task_step/computer_recording_test.py @@ -0,0 +1,202 @@ +"""start_computer_recording and end_computer_recording against a deployed env standing in for a driver: the grant +it's handed, the artifact its uploaded segments and manifest become, what's registered when the driver forgot the +recording, and teardown and reset ending what a run left open.""" + +from __future__ import annotations + +import pytest +from agentenv_protocol.computer.card import COMPUTER_RECORDING + +from agent_env.artifact import Artifact +from agent_env.config import set_object_store +from agent_env.store.base import GrantUnavailableError +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps import computer_recording as recording_steps +from agent_env.task_step.task_steps.computer_recording import ( + RECORDINGS_KEY, + EndComputerRecordingTaskStep, + StartComputerRecordingTaskStep, +) +from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep +from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep +from tst.util.granting_object_store import GrantingObjectStore + +_CARD = {"name": "desktop", "capabilities": {"extensions": [{"uri": COMPUTER_RECORDING, "params": { + "max_duration_s": 3600, "max_segment_s": 300, "max_concurrent": 2, + "methods": {"start": {"method": "POST"}, "end": {"method": "POST"}}}}]}} + + +class _Computer: + """A deployed env whose driver records: each end uploads one segment per started recording under its grant, + or leaves the last one out when ``drop_last``.""" + + def __init__(self, store, env_id: str = "desktop", forgets: bool = False, drop_last: bool = False) -> None: + self.env_id, self.environment_card, self.sandbox_type = env_id, _CARD, "local" + self.store, self.forgets, self.drop_last = store, forgets, drop_last + self.calls: list[tuple[str, dict]] = [] + + async def invoke(self, uri, method, params=None, **_): + assert uri == COMPUTER_RECORDING + self.calls.append((method, params)) + if method == "start": + root = params["upload"]["root_path"] + self.store.put(f"{root}/rec-1/0000.mp4", b"mp4-0", content_type="video/mp4") + return {"recording_id": "rec-1", "started_at": "2026-10-27T10:00:00.000Z", "format": "video/mp4"} + if self.forgets: + raise RuntimeError("400 invalid_argument: there is no recording 'rec-1' on this computer") + segments = [{"index": 0, "path": "rec-1/0000.mp4", "uploaded": True}, + {"index": 1, "path": "rec-1/0001.mp4", "uploaded": not self.drop_last}] + if not self.drop_last: + self.store.put(f"{params['upload']['root_path']}/rec-1/0001.mp4", b"mp4-1", content_type="video/mp4") + return {"recording_id": "rec-1", "ended_by": params["reason"], "segments": segments} + + +@pytest.fixture +def store(local_stores, tmp_path): + granting = GrantingObjectStore(str(tmp_path / "objects")) + set_object_store(granting) + return granting + + +def _context(*envs) -> TaskStepContext: + return TaskStepContext(instance_id="run-1", deployed_envs=list(envs), metadata={}) + + +def _start(name: str = "desk", env_id: str = "desktop", **kwargs) -> StartComputerRecordingTaskStep: + return StartComputerRecordingTaskStep(id=f"start-{name}", version=None, env_id=env_id, name=name, **kwargs) + + +def _end(name: str = "desk", **kwargs) -> EndComputerRecordingTaskStep: + return EndComputerRecordingTaskStep(id=f"end-{name}", version=None, name=name, **kwargs) + + +@pytest.mark.asyncio +async def test_a_recording_is_started_with_a_grant_and_registered_with_its_manifest_at_end(store): + computer = _Computer(store) + context = await _start(fps=5).execute(_context(computer)) + started = dict(context.metadata[RECORDINGS_KEY]["desk"]) + context = await _end().execute(context) + entry = context.metadata[RECORDINGS_KEY]["desk"] + + (_, start), (_, end) = computer.calls + assert start["request_id"] == "run-1:start-desk" and start["fps"] == 5 and start["cursor"] is True + assert "/agentenv-recordings/" in started["namespace"] and started["state"] == "open" + assert end["upload"]["root_path"] == start["upload"]["root_path"] and end["reason"] == "end" + assert (entry["state"], entry["ended_by"], entry["segments"], entry["missing_segments"]) == ("ended", "end", 2, 0) + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + assert sorted(universe.file_artifact_ids) == ["manifest.json", "rec-1-0000.mp4", "rec-1-0001.mp4"] + segment = Artifact.get(universe.file_artifact_ids["rec-1-0001.mp4"]) + assert segment.object_url.endswith("/rec-1/0001.mp4") and store.get(segment.object_url) == b"mp4-1" + assert b'"ended_by": "end"' in store.get(Artifact.get(universe.file_artifact_ids["manifest.json"]).object_url) + + +@pytest.mark.asyncio +async def test_a_segment_the_driver_couldnt_upload_is_left_out_and_counted(store): + context = await _start().execute(_context(_Computer(store, drop_last=True))) + entry = (await _end().execute(context)).metadata[RECORDINGS_KEY]["desk"] + universe = Artifact.get(entry["artifact"]["id"]) + assert (entry["segments"], entry["missing_segments"]) == (1, 1) + assert sorted(universe.file_artifact_ids) == ["manifest.json", "rec-1-0000.mp4"] + + +@pytest.mark.asyncio +async def test_a_recording_the_driver_forgot_is_registered_from_its_namespace(store): + context = await _start().execute(_context(_Computer(store, forgets=True))) + entry = (await _end().execute(context)).metadata[RECORDINGS_KEY]["desk"] + assert entry["state"] == "ended" and entry["ended_by"] == "error" and "no recording 'rec-1'" in entry["error"] + assert sorted(Artifact.get(entry["artifact"]["id"]).file_artifact_ids) == ["manifest.json", "rec-1-0000.mp4"] + + +@pytest.mark.asyncio +async def test_a_store_whose_grants_dont_reach_the_sandbox_fails_the_start_best_effort_or_fatally(store): + computer = _Computer(store) + computer.sandbox_type = "modal" + context = await _start().execute(_context(computer)) + entry = context.metadata[RECORDINGS_KEY]["desk"] + assert entry["state"] == "failed" and "reaches env desktop's sandbox (modal)" in entry["error"] + assert computer.calls == [] + await _end().execute(context) + assert context.metadata[RECORDINGS_KEY]["desk"]["state"] == "failed" + with pytest.raises(RuntimeError, match="issues no upload grant"): + await _start(fail_task_on_error=True).execute(_context(computer)) + + +@pytest.mark.asyncio +async def test_ending_a_recording_never_started_is_logged_or_fatal(store): + context = await _end().execute(_context(_Computer(store))) + assert RECORDINGS_KEY not in context.metadata + with pytest.raises(RuntimeError, match="no recording named 'desk'"): + await _end(fail_task_on_error=True).execute(context) + + +@pytest.mark.asyncio +async def test_a_fatal_end_raises_when_the_recording_couldnt_be_registered(store, monkeypatch): + context = await _start().execute(_context(_Computer(store))) + + def broken(*args): + raise RuntimeError("the document store is down") + + monkeypatch.setattr(recording_steps, "_register", broken) + with pytest.raises(RuntimeError, match="didn't end: RuntimeError: the document store is down"): + await _end(fail_task_on_error=True).execute(context) + + +@pytest.mark.asyncio +async def test_teardown_and_reset_end_what_a_run_left_open_with_their_reason(store): + desktop, other = _Computer(store), _Computer(store, env_id="other") + context = _context(desktop, other) + context = await _start("a").execute(context) + context = await _start("b", env_id="other").execute(context) + await ResetEnvTaskStep(id="reset", version=None, env_id="other").execute(context) + await teardown_run(context) + recordings = context.metadata[RECORDINGS_KEY] + assert (recordings["a"]["ended_by"], recordings["b"]["ended_by"]) == ("teardown", "reset") + assert [method for method, _ in desktop.calls] == ["start", "end"] + + +@pytest.mark.asyncio +async def test_a_teardown_sandboxes_step_ends_the_recordings_of_its_envs_only(store): + desktop, other = _Computer(store), _Computer(store, env_id="other") + context = _context(desktop, other) + context = await _start("a").execute(context) + context = await _start("b", env_id="other").execute(context) + await TeardownSandboxesTaskStep(id="down", version=None, env_ids=["desktop"]).execute(context) + recordings = context.metadata[RECORDINGS_KEY] + assert (recordings["a"]["state"], recordings["b"]["state"]) == ("ended", "open") + + +def test_the_steps_round_trip_and_refuse_what_they_cant_run(): + start = _start(fps=5, max_edge=1280, cursor=False, fail_task_on_error=True) + assert StartComputerRecordingTaskStep.from_dict(start.to_dict()).to_dict() == start.to_dict() + assert EndComputerRecordingTaskStep.from_dict(_end().to_dict()).to_dict() == _end().to_dict() + assert StartComputerRecordingTaskStep.from_dict({"id": "s", "env_id": "e", "name": "n"}).fail_task_on_error is False + with pytest.raises(ValueError, match="needs an env_id and a name"): + _start(name="") + with pytest.raises(ValueError, match="needs the name"): + _end(name="") + + +class _ShortGrants(GrantingObjectStore): + """A store whose credentials sign grants only as long as its grant lifetime.""" + + def issue_upload_policy(self, prefix_url, *, max_object_bytes, expires_in): + if expires_in > self.grant_lifetime_seconds: + raise GrantUnavailableError(f"temporary credentials can't sign {expires_in} s") + self.asked = expires_in + return super().issue_upload_policy(prefix_url, max_object_bytes=max_object_bytes, expires_in=expires_in) + + +@pytest.mark.asyncio +async def test_a_store_that_cant_sign_for_the_whole_recording_grants_what_it_can(local_stores, tmp_path): + short = _ShortGrants(str(tmp_path / "objects")) + short.grant_lifetime_seconds = 1800 + set_object_store(short) + context = await _start().execute(_context(_Computer(short))) + assert context.metadata[RECORDINGS_KEY]["desk"]["state"] == "open" and short.asked == short.grant_lifetime_seconds + + +@pytest.mark.asyncio +async def test_recording_an_env_this_run_didnt_deploy_fails_the_start(store): + context = await _start(env_id="elsewhere").execute(_context(_Computer(store))) + assert "env 'elsewhere' isn't deployed in this run" in context.metadata[RECORDINGS_KEY]["desk"]["error"] From b05248fd997e2ea02fbc64cd680d1e4076d1fd27 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:45:11 -0700 Subject: [PATCH 04/17] feat(computer): the reference driver records its display through x11grab The reference driver declares recording = Recording(max_fps=30) and a @record stream: PyAV's x11grab on its display, read in a thread of its own, its frames stamped with the wall clock they carry and the real cursor drawn when asked. A reader that falls behind gets the latest frames. The scaffold installs the recording extra. A new recording task records a scripted drive of the Fixture Desk, and the agent task records Claude Code's run. The integration test decodes the recording: every drive screenshot matches the video frame at its captured_at, within a frame, and the clicked grid cell changes between the screenshots before and after its click. The dev extra gains av, to decode with. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + .../drivers/envs/reference/driver.py | 55 ++++++++++- .../scaffold/artifacts/drive_desk/verify.py | 53 ++++++++++ .../computer/scaffold/envs/desktop/Dockerfile | 2 +- .../computer/scaffold/tasks/agent.json | 8 +- .../computer/scaffold/tasks/recording.json | 8 ++ tst/integration/computer/scaffold_test.py | 77 ++++++++++++++- tst/unit/drivers/reference_driver_test.py | 98 ++++++++++++++++++- uv.lock | 3 + 9 files changed, 292 insertions(+), 13 deletions(-) create mode 100644 tst/integration/computer/scaffold/artifacts/drive_desk/verify.py create mode 100644 tst/integration/computer/scaffold/tasks/recording.json diff --git a/pyproject.toml b/pyproject.toml index 041c2294e..4cd2f1e78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ dev = [ "agentenv-framework[aws,boxd,daytona,gcp,sail,tensorlake,tenki,vercel]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs + "av>=17", # the protocol's recording extra, whose tests the unit tier runs and which the recording test decodes with "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "tomli-w==1.2.0", # the Codex agent's config writer, which only its image installs; its tests import it "moto>=5.0.0", diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index d9cae7d54..32286f2e0 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -1,17 +1,20 @@ """The reference computer driver, for an X11 desktop: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the event loop, one at a time, on a connection checked before each call and reopened when it -drops, so the driver outlives a restarted X server. It needs ``agentenv-framework-protocol[computer]``, ``mss`` and -``python-xlib`` where it runs.""" +drops, so the driver outlives a restarted X server. It records the display through FFmpeg's ``x11grab``, by PyAV. It +needs ``agentenv-framework-protocol[computer,recording]``, ``mss`` and ``python-xlib`` where it runs.""" from __future__ import annotations import asyncio import contextlib import math +import os import threading import time from collections import OrderedDict -from typing import Any, Callable +from datetime import datetime, timezone +from typing import Any, AsyncIterator, Callable +import av import mss from Xlib import X, XK from Xlib import display as xdisplay @@ -25,6 +28,9 @@ Display, Image, ReadyCheck, + RecordedFrame, + RecordHints, + Recording, Unsupported, capture, click, @@ -38,6 +44,7 @@ pointer_position, pointer_up, press, + record, scroll, type_text, ) @@ -58,6 +65,8 @@ #: How far apart a drag's pointer moves are, in pixels. DRAG_STEP_PX = 20 DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) +#: How many recorded frames wait for the encoder before the oldest is dropped. +RECORD_QUEUE = 8 @computer_driver(name="reference", @@ -66,6 +75,7 @@ class ReferenceDriver(ComputerDriverEnvironment): buttons = ("left", "right", "middle", "back", "forward") max_click_count = 3 max_drag_points = 50 + recording = Recording(max_fps=30) def __init__(self) -> None: super().__init__() @@ -85,6 +95,45 @@ def read(c): async def grab(self) -> Image: return await asyncio.to_thread(_grab) + @record + async def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: + """The display through FFmpeg's ``x11grab``, read in a thread of its own. Its frames carry the wall clock, + so each one's time is when it was captured.""" + size = await self.screen() + loop, frames, stop = asyncio.get_running_loop(), asyncio.Queue(), threading.Event() + + def offer(item: Any) -> None: + if frames.qsize() >= RECORD_QUEUE: + frames.get_nowait() + frames.put_nowait(item) + + def post(item: Any) -> None: + with contextlib.suppress(RuntimeError): # the loop has closed: nobody is reading + loop.call_soon_threadsafe(offer, item) + + def pump() -> None: + options = {"framerate": str(hints.fps), "video_size": f"{size.width}x{size.height}", + "draw_mouse": "1" if hints.cursor else "0"} + try: + with av.open(os.environ.get("DISPLAY", ":0"), format="x11grab", options=options) as source: + for frame in source.decode(video=0): + if stop.is_set(): + return + post(RecordedFrame(frame, datetime.fromtimestamp(frame.time, timezone.utc))) + except Exception as e: + post(e) + finally: + post(None) + + threading.Thread(target=pump, name="reference-record", daemon=True).start() + try: + while (item := await frames.get()) is not None: + if isinstance(item, Exception): + raise item + yield item + finally: + stop.set() + async def ready_checks(self) -> list[ReadyCheck]: try: has = await self._read(lambda c: c.d.has_extension("XTEST")) diff --git a/tst/integration/computer/scaffold/artifacts/drive_desk/verify.py b/tst/integration/computer/scaffold/artifacts/drive_desk/verify.py new file mode 100644 index 000000000..898048e35 --- /dev/null +++ b/tst/integration/computer/scaffold/artifacts/drive_desk/verify.py @@ -0,0 +1,53 @@ +"""A scripted drive of the Fixture Desk while the desktop records: the clicks, typing and scroll a person would make, +each answered with a screenshot. A test finds each moment in the video by its send time, its screenshot's capture time +and thumbnail, and the clicked grid cell's box.""" +import asyncio +import base64 +import io +import json +from datetime import datetime, timezone + +from PIL import Image + +from agentenv_protocol.computer.conformance.client import Driver + +THUMB = (64, 40) + + +def _thumb(result): + data = next(c.data for c in result.content if c.type == "image") + return Image.open(io.BytesIO(base64.b64decode(data))).convert("L").resize(THUMB).tobytes().hex() + + +async def verify(mcp_url): + driver = await Driver.connect(mcp_url.rsplit("/mcp", 1)[0], 60) + for _ in range(60): # the Fixture Desk starts beside the driver and writes its layout once it is up + out = await driver.run("cat", "fixture/layout.json") + if out["exit_code"] == 0 and out["stdout"].strip(): + break + await asyncio.sleep(0.5) + layout = json.loads(out["stdout"]) + + def centre(box): + x, y, w, h = layout[box] + return {"x": x + w // 2, "y": y + h // 2} + + gx, gy, cell = layout["grid"][0], layout["grid"][1], layout["cell"] + clicked = [gx + 4 * cell, gy + 3 * cell, cell, cell] + steps = [("computer_click", centre("name")), ("computer_type", {"text": "Ada Lovelace"}), + ("computer_click", centre("subscribe")), + ("computer_click", {"x": clicked[0] + cell // 2, "y": clicked[1] + cell // 2}), + ("computer_scroll", {**centre("list"), "direction": "down", "amount": 10}), + ("computer_click", centre("save"))] + shots = [] + async with driver.session() as session: + await session.call_tool("computer_screenshot", {}) + for tool, args in steps: + sent_at = datetime.now(timezone.utc).isoformat() + result = await session.call_tool(tool, {**args, "screenshot": True}) + if result.isError: + return [{"id": "drove", "result": False, "description": f"{tool}: {result.content[0].text}"}] + shots.append({"step": tool, "args": args, "sent_at": sent_at, + "captured_at": result.structuredContent["frame"]["captured_at"], "thumb": _thumb(result)}) + await asyncio.sleep(0.7) + return [{"id": "drove", "result": True, "description": json.dumps({"shots": shots, "clicked_cell": clicked})}] diff --git a/tst/integration/computer/scaffold/envs/desktop/Dockerfile b/tst/integration/computer/scaffold/envs/desktop/Dockerfile index b2a3c9140..724514c88 100644 --- a/tst/integration/computer/scaffold/envs/desktop/Dockerfile +++ b/tst/integration/computer/scaffold/envs/desktop/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY protocol /opt/protocol RUN python3 -m venv /opt/driver \ - && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" + && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer,recording]" "mss>=9,<11" "python-xlib>=0.33,<0.34" RUN useradd --create-home --uid 10001 user COPY start.sh driver.py fixture.py /opt/desktop/ USER user diff --git a/tst/integration/computer/scaffold/tasks/agent.json b/tst/integration/computer/scaffold/tasks/agent.json index 25cc9b49b..f2398aace 100644 --- a/tst/integration/computer/scaffold/tasks/agent.json +++ b/tst/integration/computer/scaffold/tasks/agent.json @@ -1,8 +1,8 @@ [ {"id": "env", "type": "deploy_env", "env_id": "desktop", "sandbox_type": "local"}, {"id": "agent", "type": "deploy_agent", "a2a_agent_id": "claude-code", "env_ids": ["desktop"], "sandbox_type": "local"}, - {"id": "task", "type": "prompt_agent", "prompt_id": "task", "depends_on": ["env", "agent"], - "prompt": "You have a computer with a graphical desktop, which you use through the computer_* tools: take a screenshot to see it, then click, type, scroll and press keys. A window called Fixture Desk is open. In it: type the name Ada Lovelace into the Name field, tick the Subscribe checkbox, select Item 37 in the list on the left (scroll the list to find it), click the cell in the fourth row and fifth column of the white grid of squares, and finally click Save. Take a screenshot after each step to check it worked. Use only the computer_* tools, and reply DONE when everything is in place."}, - {"id": "check", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_desk", - "verifier_id": "desk", "depends_on": ["task"]} + {"id": "record", "type": "start_computer_recording", "env_id": "desktop", "name": "agent", "fps": 10, "depends_on": ["env"]}, + {"id": "task", "type": "prompt_agent", "prompt_id": "task", "depends_on": ["record", "agent"], "prompt": "You have a computer with a graphical desktop, which you use through the computer_* tools: take a screenshot to see it, then click, type, scroll and press keys. A window called Fixture Desk is open. In it: type the name Ada Lovelace into the Name field, tick the Subscribe checkbox, select Item 37 in the list on the left (scroll the list to find it), click the cell in the fourth row and fifth column of the white grid of squares, and finally click Save. Take a screenshot after each step to check it worked. Use only the computer_* tools, and reply DONE when everything is in place."}, + {"id": "check", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_desk", "verifier_id": "desk", "depends_on": ["task"]}, + {"id": "end", "type": "end_computer_recording", "name": "agent", "depends_on": ["task"]} ] diff --git a/tst/integration/computer/scaffold/tasks/recording.json b/tst/integration/computer/scaffold/tasks/recording.json new file mode 100644 index 000000000..a407607db --- /dev/null +++ b/tst/integration/computer/scaffold/tasks/recording.json @@ -0,0 +1,8 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop", "sandbox_type": "local"}, + {"id": "record", "type": "start_computer_recording", "env_id": "desktop", "name": "desk", "fps": 10, + "depends_on": ["env"]}, + {"id": "drive", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "drive_desk", + "verifier_id": "drive", "depends_on": ["record"]}, + {"id": "end", "type": "end_computer_recording", "name": "desk", "depends_on": ["drive"]} +] diff --git a/tst/integration/computer/scaffold_test.py b/tst/integration/computer/scaffold_test.py index e87ba4e57..01b951bca 100644 --- a/tst/integration/computer/scaffold_test.py +++ b/tst/integration/computer/scaffold_test.py @@ -1,20 +1,30 @@ """The reference driver on a desktop of its own before agent-env can compose a computer: the scaffold, a hand-written Ubuntu 24.04 image under this folder, deployed by ``agent-env run`` as an MCP server on the server provider, passes -the conformance probe at full three runs in a row, and an agent drives its desktop through the computer_* tools. -Needs a Docker daemon and the local registry; the agent's task, a model endpoint too.""" +the conformance probe at full three runs in a row, records a scripted drive of its desk that lines up with the drive's +screenshots, and an agent drives its desktop through the computer_* tools under a recording. Needs a Docker daemon and +the local registry; the agent's task, a model endpoint too.""" +import json import logging +import re import shutil +from datetime import datetime from pathlib import Path +import av import pytest from click.testing import CliRunner +from PIL import Image import agent_env import agentenv_protocol +from agent_env.artifact import Artifact from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli -from agent_env.config import configure, reset_config +from agent_env.config import configure, get_config, reset_config +from agent_env.store.object_store.local.grant_server import grant_server +from agent_env.store.routing import namespace_routing +from agent_env.task.store import get_task_instance_store from tst.util.capabilities import skip_without_model_endpoint pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -43,6 +53,7 @@ def state(monkeypatch, tmp_path): assert not sandboxes.exists() or not any(sandboxes.iterdir()), "a run left a sandbox work folder" finally: logging.disable(logging.NOTSET) + grant_server(None, None).close() reset_artifact_store() reset_config() @@ -64,6 +75,35 @@ def _run(root: Path, task: str): return CliRunner().invoke(cli, ["run", str(root), "--task", task]) +def _recorded(output: str, name: str, tmp: Path) -> tuple[dict, dict, list]: + """The run's context, the manifest of its recording ``name``, and every frame of the video with its time on the + computer's clock.""" + instance_id = re.search(r"instance (@local/\S+)", output).group(1) + with namespace_routing(): + context = get_task_instance_store().get(instance_id).context + entry = context["metadata"]["computer_recordings"][name] + universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) + objects = get_config().get_object_store() + files = {name: objects.get(Artifact.get(ref).object_url) for name, ref in universe.file_artifact_ids.items()} + manifest = json.loads(files["manifest.json"]) + frames = [] + for segment in manifest["segments"]: + path = tmp / segment["path"].replace("/", "-") + path.write_bytes(files[segment["path"].replace("/", "-")]) + with av.open(str(path)) as video: + start = _when(segment["started_at"]) + frames += [(start.timestamp() + frame.time, frame.to_image()) for frame in video.decode(video=0)] + return context, manifest, frames + + +def _when(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +def _thumb(image: Image.Image) -> bytes: + return image.convert("L").resize((64, 40)).tobytes() + + def test_the_reference_driver_passes_the_probe_at_full(state): ran = _run(_scaffold(state / "scaffold"), "probe") @@ -71,9 +111,38 @@ def test_the_reference_driver_passes_the_probe_at_full(state): assert "passed (probe: 1)" in ran.output, ran.output +def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state): + ran = _run(_scaffold(state / "scaffold"), "recording") + + assert ran.exit_code == 0, ran.output + assert "passed (drive: 1)" in ran.output, ran.output + context, manifest, frames = _recorded(ran.output, "desk", state) + entry = context["metadata"]["computer_recordings"]["desk"] + assert (entry["state"], entry["ended_by"], entry["missing_segments"]) == ("ended", "end", 0) + assert manifest["source"] == "stream" and manifest["cursor"] is True and frames + assert len(frames) == sum(s["frames"] for s in manifest["segments"]) + drive = json.loads(context["metadata"]["verifications"]["drive"]["results"][0]["description"]) + interval = 1 / manifest["fps"] + for shot in drive["shots"]: + at = _when(shot["captured_at"]).timestamp() + nearest = min(frames, key=lambda frame: abs(frame[0] - at)) + assert abs(nearest[0] - at) <= interval, (shot["step"], nearest[0] - at) + diff = sum(abs(a - b) for a, b in zip(_thumb(nearest[1]), bytes.fromhex(shot["thumb"]))) / (64 * 40) + assert diff < 6, (shot["step"], diff) + x, y, w, h = drive["clicked_cell"] + shots = drive["shots"] + n = next(i for i, s in enumerate(shots) if s["args"].get("x") == x + w // 2) + corner = lambda image: image.getpixel((x + 8, y + 8)) # noqa: E731 clear of the pointer drawn at the click + changed = next(t for t, image in frames if corner(image) != corner(frames[0][1])) + # The cell changes after the screenshot before its click and by the one after it, both on the computer's clock. + assert _when(shots[n - 1]["captured_at"]).timestamp() < changed <= _when(shots[n]["captured_at"]).timestamp() + interval + + @skip_without_model_endpoint() -def test_an_agent_drives_the_reference_drivers_desktop(state): +def test_an_agent_drives_the_reference_drivers_desktop_under_a_recording(state): ran = _run(_scaffold(state / "scaffold"), "agent") assert ran.exit_code == 0, ran.output assert "passed (check: 1)" in ran.output, ran.output + context, manifest, frames = _recorded(ran.output, "agent", state) + assert context["metadata"]["computer_recordings"]["agent"]["state"] == "ended" and frames diff --git a/tst/unit/drivers/reference_driver_test.py b/tst/unit/drivers/reference_driver_test.py index 42c63896a..cbed28526 100644 --- a/tst/unit/drivers/reference_driver_test.py +++ b/tst/unit/drivers/reference_driver_test.py @@ -7,7 +7,9 @@ import contextlib import importlib.util import sys +import time import types +from datetime import datetime, timezone from pathlib import Path import httpx @@ -15,7 +17,8 @@ from mcp.shared.memory import create_connected_server_and_client_session import agent_env -from agentenv_protocol.computer import COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER, Platform +from agentenv_protocol.computer import (COMPUTER_EXEC, COMPUTER_KEYBOARD, COMPUTER_POINTER, COMPUTER_RECORDING, + Platform, RecordHints) DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "reference" / "driver.py" LINUX = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="x11") @@ -94,6 +97,37 @@ def grab(self, monitor: dict): return types.SimpleNamespace(width=WIDTH, height=HEIGHT, bgra=bytes(WIDTH * HEIGHT * 4)) +class FakeGrab: + """FFmpeg's x11grab through PyAV: ``frames`` frames a few milliseconds apart, each with its wall-clock time, then + ``fail`` raised if set; ``opened`` keeps what each open was asked.""" + opened: list = [] + frames = 3 + fail: Exception | None = None + closed = 0 + + def __init__(self, display: str, format: str, options: dict) -> None: + self.args = (display, format, options) + + @classmethod + def open(cls, display, format, options): + grab = cls(display, format, options) + cls.opened.append(grab.args) + return grab + + def __enter__(self): + return self + + def __exit__(self, *exc): + FakeGrab.closed += 1 + + def decode(self, video): + for n in range(self.frames): + time.sleep(0.005) + yield types.SimpleNamespace(time=1_793_000_000.0 + n / 10, index=n) + if self.fail is not None: + raise self.fail + + @pytest.fixture def x11(monkeypatch): """The example's driver on a fake X display, and the keycodes the fake keymap gives named keys.""" @@ -130,6 +164,7 @@ def module(name: str, **attrs) -> types.ModuleType: module("Xlib.ext", xtest=xtest) module("Xlib", X=X, XK=XK, display=display) module("mss", MSS=FakeMSS) + module("av", open=FakeGrab.open) monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: LINUX) monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) @@ -301,3 +336,64 @@ async def test_a_dropped_x_connection_is_reopened_and_a_dead_one_is_device_unava assert not moved.isError and reopened == 2 assert refused.structuredContent["code"] == "device_unavailable" assert "the X server is unavailable: ConnectionClosedError" in refused.structuredContent["message"] + + +def _hints(tmp_path, **kwargs) -> RecordHints: + return RecordHints(**{"scope": "display", "fps": 12, "max_edge": None, "cursor": True, "max_segment_s": 300, + "directory": tmp_path, **kwargs}) + + +@pytest.fixture +def grab(monkeypatch): + monkeypatch.setattr(FakeGrab, "opened", []) + monkeypatch.setattr(FakeGrab, "frames", 3) + monkeypatch.setattr(FakeGrab, "fail", None) + monkeypatch.setattr(FakeGrab, "closed", 0) + monkeypatch.setenv("DISPLAY", ":99") + return FakeGrab + + +@pytest.mark.asyncio +async def test_recording_streams_x11grab_frames_stamped_with_their_capture_time(x11, grab, tmp_path): + driver, _fake, _code = x11 + frames = [frame async for frame in driver.stream(_hints(tmp_path, cursor=False))] + assert grab.opened == [(":99", "x11grab", {"framerate": "12", "video_size": "640x400", "draw_mouse": "0"})] + assert [f.image.index for f in frames] == [0, 1, 2] and grab.closed == 1 + assert frames[1].captured_at == datetime.fromtimestamp(1_793_000_000.1, timezone.utc) + + +@pytest.mark.asyncio +async def test_a_failed_grab_raises_from_the_stream_and_a_slow_reader_gets_the_latest_frames(x11, grab, tmp_path): + driver, _fake, _code = x11 + grab.fail = RuntimeError("Cannot open display :99") + with pytest.raises(RuntimeError, match="Cannot open display"): + [frame async for frame in driver.stream(_hints(tmp_path))] + grab.fail, grab.frames = None, 30 + stream = driver.stream(_hints(tmp_path)) + first = await anext(stream) + await asyncio.sleep(0.5) + rest = [frame async for frame in stream] + assert first.image.index == 0 and len(rest) < 10 and rest[-1].image.index == 29 + + +@pytest.mark.asyncio +async def test_closing_the_stream_stops_the_grab(x11, grab, tmp_path): + driver, _fake, _code = x11 + grab.frames = 10_000 + stream = driver.stream(_hints(tmp_path)) + await anext(stream) + await stream.aclose() + for _ in range(100): + if grab.closed: + break + await asyncio.sleep(0.01) + assert grab.closed == 1 + + +@pytest.mark.asyncio +async def test_the_reference_driver_records_from_its_stream_at_up_to_30_fps(x11): + driver, _fake, _code = x11 + async with http_client(driver) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + recording = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) + assert recording["params"]["source"] == "stream" and recording["params"]["fps"] == {"default": 10, "max": 30} diff --git a/uv.lock b/uv.lock index cbd481459..24f51ffac 100644 --- a/uv.lock +++ b/uv.lock @@ -80,6 +80,8 @@ daytona = [ ] dev = [ { name = "anyio" }, + { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "boto3" }, { name = "botocore" }, { name = "boxd" }, @@ -139,6 +141,7 @@ requires-dist = [ { name = "agentenv-framework", extras = ["aws", "boxd", "daytona", "gcp", "sail", "tensorlake", "tenki", "vercel"], marker = "extra == 'dev'" }, { name = "agentenv-framework-protocol", extras = ["computer"], editable = "packages/agentenv-protocol" }, { name = "anyio", marker = "extra == 'vercel'", specifier = ">=4,<5" }, + { name = "av", marker = "extra == 'dev'", specifier = ">=17" }, { name = "boto3", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "botocore", marker = "extra == 'aws'", specifier = ">=1.35.0" }, { name = "boxd", marker = "extra == 'boxd'", specifier = "~=0.2.17" }, From c9bdb6a160fc42b1708bdebd23a30c31ee757527 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:45:53 -0700 Subject: [PATCH 05/17] docs(computer): document computer-recording/v1, @record and the recording steps Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +-- packages/agentenv-protocol/README.md | 46 ++++++++++++++++++++++++++-- src/agent_env/drivers/README.md | 5 +-- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e70936514..2ea61018d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed; its deploy is refused until agent-env creates computers). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, whose open recordings `teardown_run`, `teardown_sandboxes` and `reset_env` end first, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `daytona` (the `daytona` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | @@ -120,7 +120,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | -| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux) aliased `reference`. It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. `tst/integration/computer/` serves it on a scaffold desktop image and runs the conformance probe and an agent against it. | +| `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`. It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. `tst/integration/computer/` serves it on a scaffold desktop image and runs the conformance probe and an agent against it. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | ## Extension points diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 578076040..a86dcc061 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -367,6 +367,7 @@ pixels), at the display's size times its scale. Input operations may return an ` | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | | `@run_in_session` | no; a Linux default | `computer-exec/v1` | | `@write_files` | no; a Linux default | `data/add` | +| `@record` | no, with `recording = Recording()` | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | The class is checked when it is defined: a missing required operation, half a group, two methods for one operation that some platform could both match, or features that don't match the @@ -398,8 +399,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool - descriptions), `computer-keyboard/v1` and `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, - request_id}`: output capped per stream, a repeated `request_id` answered with the first result). + descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, + request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and, + with `recording`, `computer-recording/v1` (below). - **The data plane.** `data/add` takes file parts that name their path, relative to the session user's home, under the `data-objects/v1` marker (`{"path": ...}`), as inline bytes, an https URL, or a `transfers.ReadObject` read grant under the marker's `read`, and writes them atomically. @@ -425,6 +427,44 @@ names the caller whose latest frame and others' input a call uses. `MyDriver.declaration(platform)` returns the card the class serves on a `Platform`, built without running the driver, its operations or its dependencies. +### Recording the display + +A driver that declares `recording` serves `computer-recording/v1`: video of the display, as MP4 +(H.264) segments, for the framework to start and end around the work it wants on record. Encoding +needs PyAV, from the `recording` extra (`pip install 'agentenv-framework-protocol[computer,recording]'`). + +```python +from agentenv_protocol.computer import RecordedFrame, Recording, record + + +class MyDriver(ComputerDriverEnvironment): + recording = Recording(max_fps=30) # max_concurrent=2, max_duration_s=3600, max_segment_s=300 + + @record + async def stream(self, hints): # hints.scope, fps, max_edge, cursor, max_segment_s, directory + async for image, captured_at in my_screen_stream(hints.fps): + yield RecordedFrame(image, captured_at) # a Pillow image or a PyAV VideoFrame +``` + +- **The stream.** `@record` yields `RecordedFrame`s from a stream the OS pushes, which the base class + scales, encodes and splits into segments, each frame timed by its `captured_at`; or + `RecordedSegment`s, files a native recorder finished under `hints.directory`, which it only hashes, + uploads and lists. The base class closes the stream at `end`. A driver without `@record` records by + the base class polling `@capture`, with a pointer drawn at `@pointer_position`: the slow path, at + most 15 fps. +- **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns + `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment + and returns the manifest. A repeated `request_id`, or a repeated `end`, answers the same. A start + past `max_concurrent` is refused `limit_exceeded`; `max_duration_s`, free disk under 512 MB or a stream + failing for 60 s end a recording by itself. +- **Segments.** A segment closes at `max_segment_s`, when the frame size changes or after a gap in the + stream. Each uploads as it closes under `start`'s `write_namespace` grant, at + `/.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no + grant they stay under `~/.agentenv-recordings/`. +- **The manifest.** Its `segments` carry each one's path, `started_at` and `ended_at` (its first and + last frames' capture times), size, frame count, byte size, SHA-256 and whether it uploaded, and + `ended_by` says why the recording ended: `end`, `reset`, `teardown`, `limit` or `error`. + ### Checking a driver: the conformance probe The probe checks a running driver against the contract through its card, tools and extensions. It @@ -447,7 +487,7 @@ The CLI exits 0 when no check failed, 1 when one did and 2 when it can't reach t | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add` | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` | At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index b75176e17..256ee7e45 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -3,11 +3,12 @@ Computer drivers that come with agent-env, each a `computer_driver` env a comput `envs/reference/` is the reference driver, for an X11 desktop on Linux (`amd64` and `arm64`): `mss` captures the screen and XTEST, through `python-xlib`, sends the input, foreground only. It serves the seven `computer_*` tools, the pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer-spaces`, `computer-keyboard`, and -`computer-exec` and `data/add` through the base class's Linux defaults. Name it from another bundle as `reference`, the +`computer-exec` and `data/add` through the base class's Linux defaults, and `computer-recording`, from FFmpeg's +`x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. A run that names it writes it: its files, and what its class declares for each platform it supports, read without -`mss` or `python-xlib` installed. It deploys inside a computer, which agent-env can't create yet, so a deploy of it is +`mss`, `python-xlib` or PyAV installed. It deploys inside a computer, which agent-env can't create yet, so a deploy of it is refused until then. `python -m agentenv_protocol.computer.conformance ` checks it, or any driver, where it runs. This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing From 8eab2327c3858b1f3317bfbbf1560dff534a6b01 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 21:59:22 -0700 Subject: [PATCH 06/17] fix(protocol): encode a stream's frames as deltas, not each one as a keyframe A decoder marks every frame it hands over a keyframe, x11grab's included, and libx264 honours the mark, so a recording from the reference driver's stream was all keyframes: 11.7 MB for 20 s of a mostly still desktop. The encoder clears the mark. PyAV is imported once, at the module's top, as None without the recording extra. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/recording.py | 17 ++++++++--------- .../tests/test_computer_recording.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index d360ec017..dfe49dd3a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -28,6 +28,11 @@ from .errors import InvalidArgument, LimitExceeded, Unsupported from .frames import to_pil +try: + import av +except ImportError: # the recording extra isn't installed: a start says so + av = None + if TYPE_CHECKING: from .runtime import ComputerRuntime @@ -133,11 +138,8 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O if sum(not r.done for r in self.running.values()) >= features.max_concurrent: raise LimitExceeded(f"{features.max_concurrent} recordings are running, the most this computer " "records at once; end one first") - try: - import av # noqa: F401 - except ImportError: - raise Unsupported("recording needs PyAV: install agentenv-framework-protocol[computer,recording]") \ - from None + if av is None: + raise Unsupported("recording needs PyAV: install agentenv-framework-protocol[computer,recording]") recording_id = f"rec-{uuid.uuid4().hex[:12]}" root = Path.home() / RECORDINGS_DIR hints = dataclasses.replace(hints, directory=root / recording_id) @@ -334,8 +336,6 @@ class _Encoder: the video's timeline is the computer's clock.""" def __init__(self, path: Path, width: int, height: int, fps: int, started_at: datetime) -> None: - import av - self.path = path self.size = (width, height) self.started_at = self.last_at = started_at @@ -350,6 +350,7 @@ def __init__(self, path: Path, width: int, height: int, fps: int, started_at: da def add(self, frame: Any, at: datetime) -> None: pts = max(int((at - self.started_at).total_seconds() * 1000), self._last_pts + 1) frame.pts, frame.time_base = pts, Fraction(1, 1000) + frame.pict_type = av.video.frame.PictureType.NONE # a decoded stream marks every frame a keyframe for packet in self.stream.encode(frame): self.container.mux(packet) self._last_pts, self.last_at = pts, at @@ -384,8 +385,6 @@ def _draw_pointer(image: PILImage.Image, x: float, y: float) -> None: def _video_frame(image: Any, max_edge: Optional[int]) -> Any: """``image`` as a ``yuv420p`` frame of even size, its longer edge at most ``max_edge``.""" - import av - frame = image if isinstance(image, av.VideoFrame) else av.VideoFrame.from_image(image.convert("RGB")) width, height = frame.width, frame.height if max_edge is not None and max(width, height) > max_edge: diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 44c7e0511..3fced18cd 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -42,13 +42,15 @@ class Recorded(FakeDriver): class Streamed(FakeDriver): - """A stream of the fake screen's frames, as a driver with its own capture stream yields them.""" + """A stream of the fake screen's frames as a decoder hands them over, each marked a keyframe, as x11grab's are.""" recording = Recording(max_fps=30, max_segment_s=60) @record async def frames(self, hints): while True: - yield RecordedFrame(self.screen.image.copy(), datetime.now(timezone.utc)) + frame = av.VideoFrame.from_image(self.screen.image.copy()) + frame.pict_type = av.video.frame.PictureType.I + yield RecordedFrame(frame, datetime.now(timezone.utc)) await asyncio.sleep(1 / hints.fps) @@ -224,6 +226,9 @@ async def test_a_stream_of_frames_keeps_each_frames_capture_time(home): frames = _decoded(home / ".agentenv-recordings" / segment["path"]) span = (_at(segment["ended_at"]) - _at(segment["started_at"])).total_seconds() assert len(frames) == segment["frames"] >= 10 and abs(frames[-1].time - span) < 0.002 + with av.open(str(home / ".agentenv-recordings" / segment["path"])) as container: + keyframes = sum(packet.is_keyframe for packet in container.demux(video=0) if packet.size) + assert keyframes <= 2, f"{keyframes} of {len(frames)} frames were encoded as keyframes" @pytest.mark.asyncio @@ -321,7 +326,7 @@ async def test_bad_calls_are_refused_with_a_code(path, body, status, code): @pytest.mark.asyncio async def test_without_pyav_a_start_is_refused_naming_the_extra(monkeypatch): - monkeypatch.setitem(__import__("sys").modules, "av", None) + monkeypatch.setattr(recording_module, "av", None) async with http_client(Recorded()) as http: refused = await http.post(START, json={"request_id": "r"}) assert refused.status_code == 501 and "agentenv-framework-protocol[computer,recording]" in refused.text From b0fcf8d1e387a9a33a6a8dcac7bbdeaece861a28 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 22:15:35 -0700 Subject: [PATCH 07/17] fix(protocol): end stalled recordings on time, keep segments sealing at end, and probe large or short recordings - max_duration_s is a wall-clock timer, so a stream that stalls, or a native recorder that hasn't closed a segment, still ends as limit and frees its slot. - A segment being sealed when end cancels the recording finishes sealing, and end waits for it. - The failing clock resets only once an item is saved, so an encoder that fails on every frame ends the recording as error after FAILING_S. - A recording that stops itself releases its encoder thread; the stopped recordings no one ended are kept to the latest KEEP_ENDED. - RETRY_S and DISK_CHECK_S name the retry delay and the disk check's interval. - The probe reads segments back in chunks under computer-exec's output cap, checks the video spans the recording with frames enough, and removes the recordings folder when it's left empty. Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 26 +++++-- .../agentenv_protocol/computer/recording.py | 56 ++++++++++++--- .../test_computer_conformance_recording.py | 32 +++++++-- .../tests/test_computer_recording.py | 70 +++++++++++++++++++ 4 files changed, 163 insertions(+), 21 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 5569cb78a..853db7a7e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -54,9 +54,13 @@ #: How long the probe records, and the frame rate it asks for. RECORD_S = 2.0 RECORD_FPS = 5 -#: Prints a file's size, SHA-256 and bytes as JSON, run with ``python3 -c`` on the computer. -READ_FILE = ("import base64,hashlib,json,sys;d=open(sys.argv[1],'rb').read();" - "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest(),'data':base64.b64encode(d).decode()}))") +#: Prints a file's size and SHA-256 as JSON, run with ``python3 -c`` on the computer. +READ_DIGEST = ("import hashlib,json,sys;d=open(sys.argv[1],'rb').read();" + "print(json.dumps({'size':len(d),'sha256':hashlib.sha256(d).hexdigest()}))") +#: Prints ``length`` bytes of a file from ``offset``, base64, under computer-exec's output cap. +READ_CHUNK = ("import base64,sys;f=open(sys.argv[1],'rb');f.seek(int(sys.argv[2]));" + "print(base64.b64encode(f.read(int(sys.argv[3]))).decode())") +CHUNK_BYTES = 512 << 10 #: The checks only ``full`` runs, recorded unverified when the fixture can't be staged. FULL_CHECKS = (("P3", "fixture colours land where the mapping says"), ("P4", "clicks show their effect"), ("P5", "refusals leave the fixture as it was"), ("P6", "typed and pressed keys arrive"), @@ -780,25 +784,35 @@ async def limited() -> str | None: with contextlib.suppress(Exception): await self.driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": recording_id}) await self.driver.run("rm", "-rf", f"{RECORDINGS_DIR}/{recording_id}") + await self.driver.run("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true") async def _segments_match(self, manifest: dict, fps: int) -> str | None: """What is wrong with a recording without a grant, read back from the computer, or None.""" segments = manifest.get("segments") or [] if manifest.get("ended_by") != "end" or not segments: return f"it ended by {manifest.get('ended_by')!r} with {len(segments)} segments" + span = (_when(segments[-1]["ended_at"]) - _when(segments[0]["started_at"])).total_seconds() + frames = sum(s["frames"] for s in segments) + if span < RECORD_S - 2 / fps or frames < fps * RECORD_S / 2: + return f"{frames} frames over {span:.2f} s of a {RECORD_S:g} s recording at {fps} fps" for before, after in zip(segments, segments[1:]): gap = (_when(after["started_at"]) - _when(before["ended_at"])).total_seconds() if gap > 2 / fps: return f"{gap:.2f} s between segments {before['index']} and {after['index']}" for segment in segments: - out = await self.driver.run("python3", "-c", READ_FILE, f"{RECORDINGS_DIR}/{segment['path']}") - if out.get("exit_code") != 0 or out.get("truncated"): + path = f"{RECORDINGS_DIR}/{segment['path']}" + out = await self.driver.run("python3", "-c", READ_DIGEST, path) + if out.get("exit_code") != 0: return f"segment {segment['index']} couldn't be read back: {out.get('stderr', '')[-200:]}" file = json.loads(out["stdout"]) if (file["size"], file["sha256"]) != (segment["bytes"], segment["sha256"]): return f"segment {segment['index']}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ f"the manifest says {segment['bytes']}, {segment['sha256'][:12]}" - mp4 = read_mp4(base64.b64decode(file["data"])) + data = b"" + while len(data) < file["size"]: + chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) + data += base64.b64decode(chunk["stdout"]) + mp4 = read_mp4(data) span = (_when(segment["ended_at"]) - _when(segment["started_at"])).total_seconds() if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment["width"], segment["height"], segment["frames"]) or abs(mp4["duration_s"] - span) > 2 / fps): diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index dfe49dd3a..93f5e03cd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -50,6 +50,10 @@ FAILING_S = 60.0 #: Frames further apart than this start a new segment. GAP_S = 2.0 +#: How long a stream that stopped or failed waits before it is started again. +RETRY_S = 1.0 +#: How often free disk is checked while frames arrive. +DISK_CHECK_S = 5.0 KEEP_ENDED = 64 @@ -135,6 +139,9 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O if request_id in self.started: return self.started[request_id] hints, grant = self._hints(features, scope, window, fps, max_edge, cursor, upload) + stopped = [rid for rid, r in self.running.items() if r.done] + for rid in stopped[:max(0, len(stopped) - KEEP_ENDED)]: # stopped by itself and never ended + self.running.pop(rid) if sum(not r.done for r in self.running.values()) >= features.max_concurrent: raise LimitExceeded(f"{features.max_concurrent} recordings are running, the most this computer " "records at once; end one first") @@ -147,6 +154,7 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O recording = _Recording(recording_id, hints, features, root, grant, self.runtime) self.running[recording_id] = recording recording.task = asyncio.create_task(recording.run()) + recording.timer = asyncio.get_running_loop().call_later(features.max_duration_s, recording.expire) answer = {"recording_id": recording_id, "started_at": _iso(recording.requested_at), "format": MP4} self.started[request_id] = answer while len(self.started) > 4 * KEEP_ENDED: @@ -203,7 +211,10 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.error: Optional[str] = None self.done = False self.task: Optional[asyncio.Task] = None + self.timer: Optional[asyncio.TimerHandle] = None + self.released: Optional[asyncio.Future] = None self.uploads: set[asyncio.Task] = set() + self.sealing: set[asyncio.Future] = set() self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"computer-{recording_id}") self._last_disk_check = 0.0 @@ -213,11 +224,11 @@ async def run(self) -> None: while True: try: async for item in self._source(): - failing_since = None if isinstance(item, RecordedSegment): await self._add_segment(item) else: await self._add_frame(item) + failing_since = None if self.ended_by is not None: return except asyncio.CancelledError: @@ -229,10 +240,25 @@ async def run(self) -> None: self.ended_by = "error" return await self._close() - await asyncio.sleep(1.0) + await asyncio.sleep(RETRY_S) finally: - await self._close() - self.done = True + try: + await self._close() + finally: + self.done = True + self.released = asyncio.ensure_future(self._release()) + + def expire(self) -> None: + """``max_duration_s`` is up, whether or not frames are arriving.""" + if not self.done: + self.ended_by = self.ended_by or "limit" + self.task.cancel() + + async def _release(self) -> None: + """Once every closing segment is sealed, the encoder's thread goes.""" + if self.sealing: + await asyncio.gather(*self.sealing, return_exceptions=True) + self.executor.shutdown(wait=False) def _source(self) -> AsyncIterator: if self.streams: @@ -240,12 +266,15 @@ def _source(self) -> AsyncIterator: return _poll(self.runtime, self.hints, self.cursor) async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: + if self.timer is not None: + self.timer.cancel() if self.task is not None and not self.task.done(): self.task.cancel() with contextlib.suppress(asyncio.CancelledError): await self.task if self.ended_by is None: self.ended_by = reason + await (self.released or asyncio.ensure_future(self._release())) if self.uploads: await asyncio.gather(*self.uploads, return_exceptions=True) uploader = NamespaceUploader(grant) if grant is not None else self.uploader @@ -253,7 +282,6 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: for segment in self.segments: if not segment["uploaded"]: await self._upload(uploader, segment) - self.executor.shutdown(wait=False) with contextlib.suppress(OSError): self.hints.directory.rmdir() first, last = (self.segments[0], self.segments[-1]) if self.segments else ({}, {}) @@ -277,7 +305,7 @@ async def _add_frame(self, item: RecordedFrame) -> None: path = self.hints.directory / f"{len(self.segments):04d}.mp4" self.encoder = await self._in_thread(_Encoder, path, frame.width, frame.height, self.hints.fps, at) await self._in_thread(self.encoder.add, frame, at) - self._check_limits(at) + self._check_disk() async def _add_segment(self, item: RecordedSegment) -> None: index = len(self.segments) @@ -285,20 +313,26 @@ async def _add_segment(self, item: RecordedSegment) -> None: if Path(item.path) != target: await asyncio.to_thread(shutil.move, str(item.path), target) await self._finish(target, item.started_at, item.ended_at, item.width, item.height, item.frames) - self._check_limits(item.ended_at) + self._check_disk() - def _check_limits(self, at: datetime) -> None: - if (at - self.requested_at).total_seconds() >= self.features.max_duration_s: - self.ended_by = "limit" - elif time.monotonic() - self._last_disk_check > 5: + def _check_disk(self) -> None: + if time.monotonic() - self._last_disk_check > DISK_CHECK_S: self._last_disk_check = time.monotonic() if shutil.disk_usage(self.root).free < MIN_FREE_BYTES: self.ended_by = "limit" async def _close(self) -> None: + """Seal the open segment. Sealing runs on even when ``end`` cancels the recording meanwhile, and ``end`` + waits for it, so a segment is never lost between its encoder and the manifest.""" encoder, self.encoder = self.encoder, None if encoder is None: return + sealing = asyncio.ensure_future(self._seal(encoder)) + self.sealing.add(sealing) + sealing.add_done_callback(self.sealing.discard) + await asyncio.shield(sealing) + + async def _seal(self, encoder: _Encoder) -> None: await self._in_thread(encoder.close) if encoder.frames: await self._finish(encoder.path, encoder.started_at, encoder.last_at, *encoder.size, encoder.frames) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index 20ecdfef6..b51a93e0c 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -3,7 +3,9 @@ every recording the probe made is ended and removed.""" from __future__ import annotations +import asyncio import struct +from datetime import datetime, timezone import pytest @@ -13,7 +15,7 @@ from probe_fake import FixtureDriver, served # noqa: E402 -from agentenv_protocol.computer import Recording, capture # noqa: E402 +from agentenv_protocol.computer import RecordedFrame, Recording, capture, record # noqa: E402 from agentenv_protocol.computer import recording as recording_module # noqa: E402 from agentenv_protocol.computer.conformance import checks, run_probe # noqa: E402 from agentenv_protocol.computer.conformance.client import Driver # noqa: E402 @@ -50,7 +52,7 @@ async def test_a_driver_that_records_passes_the_recording_checks_at_full_and_lea statuses = {c.name: c.status for c in report.checks} assert report.passed and all(statuses[name] == "passed" for name in NAMES) assert report.extensions["urn:agentenv:computer-recording/v1"] == "verified" - assert not any((home / ".agentenv-recordings").iterdir()) + assert not (home / ".agentenv-recordings").exists() @pytest.mark.asyncio @@ -73,7 +75,7 @@ async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still found = await _recording_checks(driver) assert found[NAMES[2]][0] == "failed" and "recording 2 of at most 1 started" in found[NAMES[2]][1] assert found[NAMES[0]][0] == found[NAMES[1]][0] == "passed" - assert not driver._computer.recordings.running and not any((home / ".agentenv-recordings").iterdir()) + assert not driver._computer.recordings.running and not (home / ".agentenv-recordings").exists() @pytest.mark.asyncio @@ -140,7 +142,7 @@ async def test_each_way_a_recording_can_fall_short_fails_its_check(home, monkeyp monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 1 << 62) limited = await _recording_checks(Recorder()) monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 0) - monkeypatch.setattr(checks, "READ_FILE", "import sys; sys.exit(3)") + monkeypatch.setattr(checks, "READ_DIGEST", "import sys; sys.exit(3)") unreadable = await _recording_checks(Recorder()) monkeypatch.undo() monkeypatch.setattr(checks, "RECORD_S", 0.8) @@ -167,3 +169,25 @@ async def test_a_card_with_no_max_concurrent_fails_the_limit_check(home, monkeyp if k != "max_concurrent"}) found = await _recording_checks(Recorder()) assert found[NAMES[2]] == ("failed", "the card declares no max_concurrent") + + +class Stalled(Recorder): + """A stream that hands over one frame and then nothing.""" + + @record + async def frames(self, hints): + yield RecordedFrame(self.screen.image.copy(), datetime.now(timezone.utc)) + await asyncio.sleep(3600) + + +@pytest.mark.asyncio +async def test_a_stream_that_stalls_after_its_first_frame_fails(home): + found = await _recording_checks(Stalled()) + assert found[NAMES[0]][0] == "failed" and "1 frames over 0.00 s of a 0.8 s recording" in found[NAMES[0]][1] + + +@pytest.mark.asyncio +async def test_a_segment_bigger_than_a_chunk_is_read_back_in_pieces(home, monkeypatch): + monkeypatch.setattr(checks, "CHUNK_BYTES", 1024) + found = await _recording_checks(Recorder()) + assert found[NAMES[0]] == ("passed", "") diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 3fced18cd..e7e5c70f1 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -4,6 +4,7 @@ import asyncio import hashlib +import time from datetime import datetime, timedelta, timezone from fractions import Fraction from unittest.mock import AsyncMock, patch @@ -368,3 +369,72 @@ async def test_only_the_latest_starts_and_ends_are_kept_for_retries(home, monkey await http.post(END, json={"recording_id": started["recording_id"]}) kept = driver._computer.recordings assert list(kept.started) == list("bcde") and len(kept.ended) == 1 + + +class Stalls(FakeDriver): + """A stream that hands over one frame and then waits forever.""" + recording = Recording(max_duration_s=1, max_concurrent=1) + + @record + async def frames(self, hints): + yield RecordedFrame(self.screen.image.copy(), datetime.now(timezone.utc)) + await asyncio.sleep(3600) + + +@pytest.mark.asyncio +async def test_a_stalled_stream_still_ends_at_max_duration_and_frees_its_slot(home): + driver = Stalls() + async with http_client(driver) as http: + first = (await http.post(START, json={"request_id": "a"})).json() + await asyncio.sleep(1.4) + second = await http.post(START, json={"request_id": "b"}) + ended = (await http.post(END, json={"recording_id": first["recording_id"]})).json() + await http.post(END, json={"recording_id": second.json()["recording_id"]}) + assert second.status_code == 200 and ended["ended_by"] == "limit" and len(ended["segments"]) == 1 + + +@pytest.mark.asyncio +async def test_an_encoder_that_fails_every_frame_ends_the_recording_with_its_error(home, monkeypatch): + monkeypatch.setattr(recording_module, "FAILING_S", 0.5) + monkeypatch.setattr(recording_module, "RETRY_S", 0.1) + + def fail(self, frame, at): + raise RuntimeError("the encoder is out of memory") + + monkeypatch.setattr(recording_module._Encoder, "add", fail) + _, manifest = await _record(Recorded(), 1.5) + assert manifest["ended_by"] == "error" and manifest["error"] == "RuntimeError: the encoder is out of memory" + + +@pytest.mark.asyncio +async def test_an_end_that_comes_while_a_segment_is_sealing_keeps_it(home, monkeypatch): + real_close = recording_module._Encoder.close + + def slow_close(self): + time.sleep(0.6) + real_close(self) + + monkeypatch.setattr(recording_module._Encoder, "close", slow_close) + _, manifest = await _record(Recorded(), 1.2) + (segment,) = manifest["segments"] # the end came while the rolled-over segment was still sealing + assert segment["frames"] >= 8 + for segment in manifest["segments"]: + assert len(_decoded(home / ".agentenv-recordings" / segment["path"])) == segment["frames"] + + +@pytest.mark.asyncio +async def test_recordings_that_stop_themselves_release_their_thread_and_only_the_latest_are_kept(home, monkeypatch): + monkeypatch.setattr(recording_module, "KEEP_ENDED", 1) + driver = FakeDriver() + driver.recording = Recording(max_duration_s=1, max_concurrent=4) + async with http_client(driver) as http: + for name in "abc": + await http.post(START, json={"request_id": name}) + await asyncio.sleep(1.6) + stopped = list(driver._computer.recordings.running.values()) + await http.post(START, json={"request_id": "d"}) + running = driver._computer.recordings.running + assert all(r.done and r.executor._shutdown for r in stopped) + assert len(running) == 2 and sum(not r.done for r in running.values()) == 1 + for recording_id in list(running): + await http.post(END, json={"recording_id": recording_id}) From 937dd08ec1ae3b89302bf4e33fdd093bf1456c62 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 22:15:35 -0700 Subject: [PATCH 08/17] fix(task): keep a retried recording's namespace, refuse a reused name, and never let a recording block teardown - A recording's namespace is named by its start's request id, so a retried start_computer_recording joins the first attempt's recording and its uploads. - A start under the name of a recording still open is refused, and the open one is left alone. - A recording whose driver forgot it and whose namespace can't be listed is marked failed; ending the open recordings never raises, so teardown always goes on. - A grant takes MAX_SEGMENTS (1000) segments and MAX_RECORDING_BYTES, as a recording also rolls over early on size changes and gaps. Co-Authored-By: Claude Opus 5.5 --- .../task_steps/computer_recording.py | 65 +++++++++++++------ tst/unit/task_step/computer_recording_test.py | 60 ++++++++++++++++- 2 files changed, 103 insertions(+), 22 deletions(-) diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 0f155c888..966a8142f 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -32,6 +32,9 @@ #: How much longer than the env's ``max_duration_s`` a recording's grant lasts. GRANT_MARGIN_S = 600 _MAX_SEGMENT_BYTES = 5 << 30 +#: The segments a grant takes: a recording also rolls over early, on a size change or after a gap. +MAX_SEGMENTS = 1000 +MAX_RECORDING_BYTES = 64 << 30 class StartComputerRecordingTaskStep(TaskStep): @@ -60,20 +63,30 @@ def from_dict(cls, data: dict) -> StartComputerRecordingTaskStep: max_edge=data.get("max_edge"), cursor=data.get("cursor", True)) async def execute(self, context: TaskStepContext) -> TaskStepContext: + recordings = context.metadata.setdefault(RECORDINGS_KEY, {}) + if (recordings.get(self.name) or {}).get("state") == "open": + error = RuntimeError(f"a recording named {self.name!r} is already open in this run; give this one " + "another name") + logger.warning("start_computer_recording: %s", error) + if self.fail_task_on_error: + raise error + return context + # A retried step joins its first attempt's recording, so it keeps that attempt's namespace too. + request_id = f"{context.instance_id}:{self.id}" if context.instance_id else uuid.uuid4().hex try: deployed = _deployed(context, self.env_id) - namespace, grant = _grant(deployed) + namespace = _namespace(request_id) + grant = _grant(deployed, namespace) started = await deployed.invoke(COMPUTER_RECORDING, "start", { - "request_id": f"{context.instance_id}:{self.id}", "fps": self.fps, "max_edge": self.max_edge, - "cursor": self.cursor, "upload": grant}) + "request_id": request_id, "fps": self.fps, "max_edge": self.max_edge, "cursor": self.cursor, + "upload": grant}) except Exception as e: - context.metadata.setdefault(RECORDINGS_KEY, {})[self.name] = { - "env_id": self.env_id, "state": "failed", "error": f"{type(e).__name__}: {e}"[:500]} + recordings[self.name] = {"env_id": self.env_id, "state": "failed", "error": f"{type(e).__name__}: {e}"[:500]} logger.warning("start_computer_recording %s on %s: %s", self.name, self.env_id, e) if self.fail_task_on_error: raise return context - context.metadata.setdefault(RECORDINGS_KEY, {})[self.name] = { + recordings[self.name] = { "env_id": self.env_id, "recording_id": started["recording_id"], "started_at": started["started_at"], "namespace": namespace, "state": "open"} logger.info("start_computer_recording %s on %s: %s", self.name, self.env_id, started["recording_id"]) @@ -120,7 +133,11 @@ async def end_open_recordings(context: TaskStepContext, reason: str, env_ids: Op """End and register every recording still open in ``context`` (of ``env_ids`` only, when given).""" for name, entry in list((context.metadata.get(RECORDINGS_KEY) or {}).items()): if entry.get("state") == "open" and (env_ids is None or entry["env_id"] in env_ids): - await end_recording(context, name, reason) + try: + await end_recording(context, name, reason) + except Exception as e: # what tears the computer down goes on regardless + entry["state"], entry["error"] = "failed", f"{type(e).__name__}: {e}"[:500] + logger.warning("ending recording %s: %s", name, e) async def end_recording(context: TaskStepContext, name: str, reason: str) -> None: @@ -131,13 +148,17 @@ async def end_recording(context: TaskStepContext, name: str, reason: str) -> Non return try: deployed = _deployed(context, entry["env_id"]) - _, grant = _grant(deployed, entry["namespace"]) manifest = await deployed.invoke(COMPUTER_RECORDING, "end", { - "recording_id": entry["recording_id"], "upload": grant, "reason": reason}) + "recording_id": entry["recording_id"], "upload": _grant(deployed, entry["namespace"]), "reason": reason}) except Exception as e: logger.warning("end_computer_recording %s: %s; registering what its namespace holds", name, e) entry["error"] = f"{type(e).__name__}: {e}"[:500] - manifest = _from_namespace(entry) + try: + manifest = _from_namespace(entry) + except Exception as listing: + logger.warning("end_computer_recording %s: listing its namespace failed: %s", name, listing) + entry["state"], entry["error"] = "failed", f"{entry['error']}; then {type(listing).__name__}: {listing}"[:500] + return try: entry["artifact"] = _register(context, name, entry["namespace"], manifest) except Exception as e: @@ -157,26 +178,30 @@ def _deployed(context: TaskStepContext, env_id: str): return deployed -def _grant(deployed, namespace: Optional[str] = None) -> tuple[str, dict]: - """A namespace for the recording's segments (a new one unless ``namespace``) and a grant that uploads under it, - lasting the env's ``max_duration_s`` and a margin, or the longest the store issues.""" +def _namespace(request_id: str) -> str: + """Where a recording's segments go: named by its start's request id, so a retried start uploads alongside.""" + config = get_config() + key = uuid.uuid5(uuid.NAMESPACE_URL, request_id).hex + return config.get_object_store().object_url(f"{config.get_artifact_key_prefix()}{_KEY_PREFIX}/{key}") + + +def _grant(deployed, namespace: str) -> dict: + """A grant that uploads under ``namespace``, lasting the env's ``max_duration_s`` and a margin, or the longest + the store issues.""" params = (find_extension(deployed.environment_card or {}, COMPUTER_RECORDING) or {}).get("params") or {} - store, config = get_config().get_object_store(), get_config() + store = get_config().get_object_store() sandbox_type = getattr(deployed, "sandbox_type", None) if not issues_grants_to(store, sandbox_type): raise RuntimeError(f"the object store issues no upload grant that reaches env {deployed.env_id}'s sandbox " f"({sandbox_type or 'unknown'}), so its recording couldn't upload") - if namespace is None: - namespace = store.object_url(f"{config.get_artifact_key_prefix()}{_KEY_PREFIX}/{uuid.uuid4().hex}") - duration, segment = params.get("max_duration_s", 3600), params.get("max_segment_s", 300) + duration = params.get("max_duration_s", 3600) cap = min(store.max_single_upload_bytes or _MAX_SEGMENT_BYTES, _MAX_SEGMENT_BYTES) - limits = ObjectLimits(max_objects=duration // segment + 2, max_object_bytes=cap, - max_total_bytes=cap * (duration // segment + 2)) + limits = ObjectLimits(max_objects=MAX_SEGMENTS, max_object_bytes=cap, max_total_bytes=MAX_RECORDING_BYTES) try: grant = namespace_grant(store, namespace, limits=limits, expires_in=duration + GRANT_MARGIN_S) except GrantUnavailableError: grant = namespace_grant(store, namespace, limits=limits, expires_in=store.grant_lifetime_seconds) - return namespace, grant.model_dump(mode="json") + return grant.model_dump(mode="json") def _from_namespace(entry: dict) -> dict: diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 6b8200749..eb792ce57 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -35,13 +35,15 @@ def __init__(self, store, env_id: str = "desktop", forgets: bool = False, drop_l self.env_id, self.environment_card, self.sandbox_type = env_id, _CARD, "local" self.store, self.forgets, self.drop_last = store, forgets, drop_last self.calls: list[tuple[str, dict]] = [] + self.started: set[str] = set() async def invoke(self, uri, method, params=None, **_): assert uri == COMPUTER_RECORDING self.calls.append((method, params)) if method == "start": - root = params["upload"]["root_path"] - self.store.put(f"{root}/rec-1/0000.mp4", b"mp4-0", content_type="video/mp4") + if params["request_id"] not in self.started: # a repeated request_id joins the first recording + self.started.add(params["request_id"]) + self.store.put(f"{params['upload']['root_path']}/rec-1/0000.mp4", b"mp4-0", content_type="video/mp4") return {"recording_id": "rec-1", "started_at": "2026-10-27T10:00:00.000Z", "format": "video/mp4"} if self.forgets: raise RuntimeError("400 invalid_argument: there is no recording 'rec-1' on this computer") @@ -200,3 +202,57 @@ async def test_a_store_that_cant_sign_for_the_whole_recording_grants_what_it_can async def test_recording_an_env_this_run_didnt_deploy_fails_the_start(store): context = await _start(env_id="elsewhere").execute(_context(_Computer(store))) assert "env 'elsewhere' isn't deployed in this run" in context.metadata[RECORDINGS_KEY]["desk"]["error"] + + +@pytest.mark.asyncio +async def test_a_retried_start_joins_the_first_attempts_namespace_and_the_grant_budgets_early_rollovers(store): + computer = _Computer(store) + context = await _start().execute(_context(computer)) + first = context.metadata[RECORDINGS_KEY].pop("desk") + context = await _start().execute(context) + (_, one), (_, two) = computer.calls + assert one["request_id"] == two["request_id"] and one["upload"]["root_path"] == two["upload"]["root_path"] + assert context.metadata[RECORDINGS_KEY]["desk"]["namespace"] == first["namespace"] + assert one["upload"]["max_objects"] == recording_steps.MAX_SEGMENTS + + +@pytest.mark.asyncio +async def test_a_start_reusing_an_open_recordings_name_is_refused_and_leaves_it_alone(store): + computer = _Computer(store) + context = await _start().execute(_context(computer)) + opened = dict(context.metadata[RECORDINGS_KEY]["desk"]) + context = await _start(env_id="desktop").execute(context) + assert context.metadata[RECORDINGS_KEY]["desk"] == opened and len(computer.calls) == 1 + with pytest.raises(RuntimeError, match="a recording named 'desk' is already open"): + await _start(fail_task_on_error=True).execute(context) + + +@pytest.mark.asyncio +async def test_teardown_goes_on_when_a_forgotten_recordings_namespace_cant_be_listed(store, monkeypatch): + context = await _start().execute(_context(_Computer(store, forgets=True))) + + def unlisted(prefix): + raise ConnectionError("the object store is down") + + monkeypatch.setattr(store, "list_at", unlisted) + await teardown_run(context) + entry = context.metadata[RECORDINGS_KEY]["desk"] + assert entry["state"] == "failed" and "then ConnectionError: the object store is down" in entry["error"] + + +@pytest.mark.asyncio +async def test_a_recording_that_cant_be_ended_at_all_is_marked_and_the_rest_still_end(store, monkeypatch): + context = _context(_Computer(store), _Computer(store, env_id="other")) + context = await _start("a").execute(context) + context = await _start("b", env_id="other").execute(context) + real = recording_steps.end_recording + + async def broken(context, name, reason): + if name == "a": + raise RuntimeError("unexpected") + await real(context, name, reason) + + monkeypatch.setattr(recording_steps, "end_recording", broken) + await recording_steps.end_open_recordings(context, "teardown") + recordings = context.metadata[RECORDINGS_KEY] + assert (recordings["a"]["state"], recordings["b"]["state"]) == ("failed", "ended") From 57792e9d79635999db66aa31a38e58deb1ec3dd4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 22:26:04 -0700 Subject: [PATCH 09/17] fix(computer): fail the probe on a segment read that returns nothing A chunk read that failed or came back empty retried the same offset forever. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/conformance/checks.py | 6 +++++- .../tests/test_computer_conformance_recording.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 853db7a7e..0f5163767 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -811,7 +811,11 @@ async def _segments_match(self, manifest: dict, fps: int) -> str | None: data = b"" while len(data) < file["size"]: chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) - data += base64.b64decode(chunk["stdout"]) + read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" + if not read: + return f"segment {segment['index']} stopped reading back at byte {len(data)} of {file['size']}: " \ + f"{chunk.get('stderr', '')[-200:]}" + data += read mp4 = read_mp4(data) span = (_when(segment["ended_at"]) - _when(segment["started_at"])).total_seconds() if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment["width"], segment["height"], segment["frames"]) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index b51a93e0c..0774556e7 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -142,8 +142,12 @@ async def test_each_way_a_recording_can_fall_short_fails_its_check(home, monkeyp monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 1 << 62) limited = await _recording_checks(Recorder()) monkeypatch.setattr(recording_module, "MIN_FREE_BYTES", 0) + real_digest = checks.READ_DIGEST monkeypatch.setattr(checks, "READ_DIGEST", "import sys; sys.exit(3)") unreadable = await _recording_checks(Recorder()) + monkeypatch.setattr(checks, "READ_DIGEST", real_digest) + monkeypatch.setattr(checks, "READ_CHUNK", "import sys; sys.exit(3)") + truncated = await _recording_checks(Recorder()) monkeypatch.undo() monkeypatch.setattr(checks, "RECORD_S", 0.8) real_end = recording_module.Recordings.end @@ -158,6 +162,7 @@ async def miscounted(self, recording_id, upload, reason): assert "s between segments 0 and 1" in gap[NAMES[0]][1] assert "it ended by 'limit'" in limited[NAMES[0]][1] assert "couldn't be read back" in unreadable[NAMES[0]][1] + assert "stopped reading back at byte 0" in truncated[NAMES[0]][1] assert "the manifest says 1280x800" in miscount[NAMES[0]][1] From 1945d26b993356ace9877a273feecbfbf6619381 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 08:10:47 -0700 Subject: [PATCH 10/17] feat(computer): let a recording run for up to 24 hours by default Computer-use tasks can run for many hours, and an hour cut them short. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 2 +- .../src/agentenv_protocol/computer/recording.py | 2 +- packages/agentenv-protocol/tests/test_computer_recording.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index a86dcc061..ed302d3f7 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -438,7 +438,7 @@ from agentenv_protocol.computer import RecordedFrame, Recording, record class MyDriver(ComputerDriverEnvironment): - recording = Recording(max_fps=30) # max_concurrent=2, max_duration_s=3600, max_segment_s=300 + recording = Recording(max_fps=30) # max_concurrent=2, max_duration_s=86400, max_segment_s=300 @record async def stream(self, hints): # hints.scope, fps, max_edge, cursor, max_segment_s, directory diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 93f5e03cd..7a7c3b8d4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -62,7 +62,7 @@ class Recording: """A driver's recording features, as a class attribute: ``recording = Recording()`` serves computer-recording/v1.""" max_fps: int = 15 max_concurrent: int = 2 - max_duration_s: int = 3600 + max_duration_s: int = 86_400 max_segment_s: int = 300 diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index e7e5c70f1..5669e6b66 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -117,7 +117,7 @@ async def test_a_driver_with_recording_serves_it_and_one_without_does_not(): assert set(extension["params"]["methods"]) == {"start", "end"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, - "cursor": True, "source": "capture", "max_segment_s": 1, "max_concurrent": 2, "max_duration_s": 3600} + "cursor": True, "source": "capture", "max_segment_s": 1, "max_concurrent": 2, "max_duration_s": 86_400} assert COMPUTER_RECORDING not in [e["uri"] for e in plain["capabilities"]["extensions"]] From 47f0dbecb807dd0f4c713266be28229168d1e89e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 08:46:22 -0700 Subject: [PATCH 11/17] feat(computer): close a recording's segments every 15 s, up to 10,000 of them A segment only plays and uploads once it closes, so the first one now lands about 15 s into a recording instead of 5 minutes, and a crash loses at most 15 s. A 24-hour recording is 5,760 segments, so a grant now takes 10,000. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 2 +- .../src/agentenv_protocol/computer/recording.py | 2 +- src/agent_env/task_step/task_steps/computer_recording.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index ed302d3f7..bf07d8ec8 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -438,7 +438,7 @@ from agentenv_protocol.computer import RecordedFrame, Recording, record class MyDriver(ComputerDriverEnvironment): - recording = Recording(max_fps=30) # max_concurrent=2, max_duration_s=86400, max_segment_s=300 + recording = Recording(max_fps=30) # max_concurrent=2, max_duration_s=86400, max_segment_s=15 @record async def stream(self, hints): # hints.scope, fps, max_edge, cursor, max_segment_s, directory diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 7a7c3b8d4..237cc9cdf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -63,7 +63,7 @@ class Recording: max_fps: int = 15 max_concurrent: int = 2 max_duration_s: int = 86_400 - max_segment_s: int = 300 + max_segment_s: int = 15 @dataclass(frozen=True) diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 966a8142f..15c29cedd 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -33,7 +33,7 @@ GRANT_MARGIN_S = 600 _MAX_SEGMENT_BYTES = 5 << 30 #: The segments a grant takes: a recording also rolls over early, on a size change or after a gap. -MAX_SEGMENTS = 1000 +MAX_SEGMENTS = 10_000 MAX_RECORDING_BYTES = 64 << 30 From 0c333caba18d342bbf58d94e52b7349672a0586f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 08:46:32 -0700 Subject: [PATCH 12/17] refactor(computer): a typed recording manifest, frames_in_thread, and named encoder settings - RecordingManifest and ManifestSegment define the manifest once, for the SDK that builds it, the probe that checks it and core that registers it. A manifest core rebuilds from a namespace knows only paths, so the measured fields are optional; the probe fails a driver's manifest that lacks them, and core falls back to the namespace when a driver's isn't one. - frames_in_thread turns a blocking frame source into a @record stream: a thread of its own, the latest 8 frames held, its errors raised, closed when the recording stops. The reference driver's @record is now a generator over x11grab. - The libx264 options, keyframe interval, max_edge bounds and kept starts are named constants, and core reads max_duration_s from the card without a stale fallback. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 15 ++- .../agentenv_protocol/computer/__init__.py | 4 +- .../computer/conformance/checks.py | 44 +++++--- .../agentenv_protocol/computer/manifest.py | 50 +++++++++ .../agentenv_protocol/computer/recording.py | 103 +++++++++++++----- .../test_computer_conformance_recording.py | 21 ++-- .../tests/test_computer_recording.py | 43 +++++++- .../drivers/envs/reference/driver.py | 48 ++------ .../task_steps/computer_recording.py | 32 +++--- tst/unit/task_step/computer_recording_test.py | 16 ++- 10 files changed, 260 insertions(+), 116 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index bf07d8ec8..fd6ac22ab 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -449,9 +449,11 @@ class MyDriver(ComputerDriverEnvironment): - **The stream.** `@record` yields `RecordedFrame`s from a stream the OS pushes, which the base class scales, encodes and splits into segments, each frame timed by its `captured_at`; or `RecordedSegment`s, files a native recorder finished under `hints.directory`, which it only hashes, - uploads and lists. The base class closes the stream at `end`. A driver without `@record` records by - the base class polling `@capture`, with a pointer drawn at `@pointer_position`: the slow path, at - most 15 fps. + uploads and lists. The base class closes the stream at `end`. A source that blocks, as most OS + capture APIs do, goes through `frames_in_thread(frames)`: the generator `frames()` runs in a thread + of its own, the latest 8 frames wait for the encoder, and it is closed when the recording stops. A + driver without `@record` records by the base class polling `@capture`, with a pointer drawn at + `@pointer_position`: the slow path, at most 15 fps. - **The methods.** `start {request_id, scope?, fps?, max_edge?, cursor?, upload?}` returns `{recording_id, started_at, format}`; `end {recording_id, upload?, reason?}` closes the last segment and returns the manifest. A repeated `request_id`, or a repeated `end`, answers the same. A start @@ -461,9 +463,10 @@ class MyDriver(ComputerDriverEnvironment): stream. Each uploads as it closes under `start`'s `write_namespace` grant, at `/.mp4`, and is then deleted; `end`'s own grant uploads any that missed. With no grant they stay under `~/.agentenv-recordings/`. -- **The manifest.** Its `segments` carry each one's path, `started_at` and `ended_at` (its first and - last frames' capture times), size, frame count, byte size, SHA-256 and whether it uploaded, and - `ended_by` says why the recording ended: `end`, `reset`, `teardown`, `limit` or `error`. +- **The manifest** (`RecordingManifest`). Its `segments` (`ManifestSegment`) carry each one's path, + `started_at` and `ended_at` (its first and last frames' capture times), size, frame count, byte size, + SHA-256 and whether it uploaded, and `ended_by` says why the recording ended: `end`, `reset`, + `teardown`, `limit` or `error`. A field it doesn't know is left out. ### Checking a driver: the conformance probe diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index de519e2d0..66afa4c69 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -52,8 +52,9 @@ type_text, write_files, ) +from .manifest import ManifestSegment, RecordingManifest from .platform import Platform -from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording +from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread from .settings import SETTINGS_ENV, ComputerSettings __all__ = [ @@ -62,6 +63,7 @@ "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", + "frames_in_thread", "RecordingManifest", "ManifestSegment", "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed", "Internal", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 0f5163767..bb5da2b9e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -16,6 +16,7 @@ from mcp import ClientSession from mcp.types import CallToolResult +from pydantic import ValidationError from ... import client as protocol from ...client import ExtensionCallError @@ -33,6 +34,7 @@ GROUP_TOOLS, POINT_TOOLS, ) +from ..manifest import RecordingManifest from ..recording import RECORDINGS_DIR from .client import Driver, image from .mp4 import read_mp4 @@ -786,42 +788,48 @@ async def limited() -> str | None: await self.driver.run("rm", "-rf", f"{RECORDINGS_DIR}/{recording_id}") await self.driver.run("sh", "-c", f"rmdir {RECORDINGS_DIR} 2>/dev/null || true") - async def _segments_match(self, manifest: dict, fps: int) -> str | None: + async def _segments_match(self, wire: dict, fps: int) -> str | None: """What is wrong with a recording without a grant, read back from the computer, or None.""" - segments = manifest.get("segments") or [] - if manifest.get("ended_by") != "end" or not segments: - return f"it ended by {manifest.get('ended_by')!r} with {len(segments)} segments" - span = (_when(segments[-1]["ended_at"]) - _when(segments[0]["started_at"])).total_seconds() - frames = sum(s["frames"] for s in segments) + try: + manifest = RecordingManifest.model_validate(wire) + except ValidationError as e: + return f"its manifest isn't a computer-recording/v1 manifest: {e}" + segments = manifest.segments + if manifest.ended_by != "end" or not segments: + return f"it ended by {manifest.ended_by!r} with {len(segments)} segments" + if unmeasured := [s.index for s in segments if not s.measured]: + return f"segments {unmeasured} lack their times, size, frame count or digest" + span = (_when(segments[-1].ended_at) - _when(segments[0].started_at)).total_seconds() + frames = sum(s.frames for s in segments) if span < RECORD_S - 2 / fps or frames < fps * RECORD_S / 2: return f"{frames} frames over {span:.2f} s of a {RECORD_S:g} s recording at {fps} fps" for before, after in zip(segments, segments[1:]): - gap = (_when(after["started_at"]) - _when(before["ended_at"])).total_seconds() + gap = (_when(after.started_at) - _when(before.ended_at)).total_seconds() if gap > 2 / fps: - return f"{gap:.2f} s between segments {before['index']} and {after['index']}" + return f"{gap:.2f} s between segments {before.index} and {after.index}" for segment in segments: - path = f"{RECORDINGS_DIR}/{segment['path']}" + path = f"{RECORDINGS_DIR}/{segment.path}" out = await self.driver.run("python3", "-c", READ_DIGEST, path) if out.get("exit_code") != 0: - return f"segment {segment['index']} couldn't be read back: {out.get('stderr', '')[-200:]}" + return f"segment {segment.index} couldn't be read back: {out.get('stderr', '')[-200:]}" file = json.loads(out["stdout"]) - if (file["size"], file["sha256"]) != (segment["bytes"], segment["sha256"]): - return f"segment {segment['index']}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ - f"the manifest says {segment['bytes']}, {segment['sha256'][:12]}" + if (file["size"], file["sha256"]) != (segment.bytes, segment.sha256): + return f"segment {segment.index}'s file is {file['size']} bytes, sha256 {file['sha256'][:12]}; " \ + f"the manifest says {segment.bytes}, {segment.sha256[:12]}" data = b"" while len(data) < file["size"]: chunk = await self.driver.run("python3", "-c", READ_CHUNK, path, str(len(data)), str(CHUNK_BYTES)) read = base64.b64decode(chunk["stdout"]) if chunk.get("exit_code") == 0 else b"" if not read: - return f"segment {segment['index']} stopped reading back at byte {len(data)} of {file['size']}: " \ + return f"segment {segment.index} stopped reading back at byte {len(data)} of {file['size']}: " \ f"{chunk.get('stderr', '')[-200:]}" data += read mp4 = read_mp4(data) - span = (_when(segment["ended_at"]) - _when(segment["started_at"])).total_seconds() - if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment["width"], segment["height"], segment["frames"]) + span = (_when(segment.ended_at) - _when(segment.started_at)).total_seconds() + if ((mp4["width"], mp4["height"], mp4["frames"]) != (segment.width, segment.height, segment.frames) or abs(mp4["duration_s"] - span) > 2 / fps): - return f"segment {segment['index']}'s MP4 is {mp4}; the manifest says {segment['width']}x" \ - f"{segment['height']}, {segment['frames']} frames over {span:.2f} s" + return f"segment {segment.index}'s MP4 is {mp4}; the manifest says {segment.width}x" \ + f"{segment.height}, {segment.frames} frames over {span:.2f} s" return None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py new file mode 100644 index 000000000..a5b851d97 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/manifest.py @@ -0,0 +1,50 @@ +"""A computer-recording/v1 manifest, as ``end`` returns it and a recording's artifact keeps it. A manifest rebuilt from +what a recording's namespace holds, for a driver that forgot the recording, knows only each segment's index and path, +so the rest is optional here; a driver's own manifest has all of it.""" +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic import BaseModel + +EndedBy = Literal["end", "reset", "teardown", "limit", "error"] + + +class ManifestSegment(BaseModel): + """One segment: ``started_at`` and ``ended_at`` are its first and last frames' capture times.""" + + index: int + path: str + started_at: Optional[str] = None + ended_at: Optional[str] = None + width: Optional[int] = None + height: Optional[int] = None + frames: Optional[int] = None + bytes: Optional[int] = None + sha256: Optional[str] = None + uploaded: bool + upload_error: Optional[str] = None + + @property + def measured(self) -> bool: + """Whether its times, size, frame count and digest are all known.""" + return None not in (self.started_at, self.ended_at, self.width, self.height, self.frames, self.bytes, + self.sha256) + + +class RecordingManifest(BaseModel): + recording_id: Optional[str] + scope: Optional[str] = None + format: Optional[str] = None + fps: Optional[int] = None + cursor: Optional[bool] = None + source: Optional[Literal["stream", "capture"]] = None + started_at: Optional[str] = None + ended_at: Optional[str] = None + ended_by: EndedBy + segments: list[ManifestSegment] + error: Optional[str] = None + + def wire(self) -> dict: + """As JSON goes on the wire, without the fields it doesn't know.""" + return self.model_dump(mode="json", exclude_none=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 237cc9cdf..6ce916e0e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -9,10 +9,11 @@ import dataclasses import hashlib import shutil +import threading import time import uuid from collections import OrderedDict -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime, timezone @@ -27,6 +28,7 @@ from ..transfers import NamespaceUploader, WriteNamespaceGrant from .errors import InvalidArgument, LimitExceeded, Unsupported from .frames import to_pil +from .manifest import ManifestSegment, RecordingManifest try: import av @@ -55,6 +57,13 @@ #: How often free disk is checked while frames arrive. DISK_CHECK_S = 5.0 KEEP_ENDED = 64 +KEEP_STARTED = 4 * KEEP_ENDED +MIN_EDGE, MAX_EDGE = 64, 8192 +#: libx264 tuned for live capture: quick to encode, at a quality screen text survives. +H264_OPTIONS = {"preset": "ultrafast", "tune": "zerolatency", "crf": "28"} +KEYFRAME_S = 2 +#: The frames ``frames_in_thread`` holds for an encoder that falls behind; past it the oldest is dropped. +THREAD_QUEUE = 8 @dataclass(frozen=True) @@ -157,7 +166,7 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O recording.timer = asyncio.get_running_loop().call_later(features.max_duration_s, recording.expire) answer = {"recording_id": recording_id, "started_at": _iso(recording.requested_at), "format": MP4} self.started[request_id] = answer - while len(self.started) > 4 * KEEP_ENDED: + while len(self.started) > KEEP_STARTED: self.started.popitem(last=False) return answer @@ -187,8 +196,8 @@ def _hints(self, features: Recording, scope: str, window: Optional[str], fps: Op fps = params["fps"]["default"] if fps is None else fps if not (isinstance(fps, int) and 1 <= fps <= params["fps"]["max"]): raise InvalidArgument(f"fps is an integer from 1 to {params['fps']['max']}, got {fps!r}") - if max_edge is not None and not (isinstance(max_edge, int) and 64 <= max_edge <= 8192): - raise InvalidArgument(f"max_edge is an integer from 64 to 8192, got {max_edge!r}") + if max_edge is not None and not (isinstance(max_edge, int) and MIN_EDGE <= max_edge <= MAX_EDGE): + raise InvalidArgument(f"max_edge is an integer from {MIN_EDGE} to {MAX_EDGE}, got {max_edge!r}") grant = _grant(upload) if upload is not None else None return RecordHints(scope, fps, max_edge, bool(cursor), features.max_segment_s, Path()), grant @@ -205,7 +214,7 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r self.requested_at = datetime.now(timezone.utc) self.streams = runtime.has("record") self.cursor = hints.cursor if self.streams else hints.cursor and runtime.has("pointer") - self.segments: list[dict] = [] + self.segments: list[_Segment] = [] self.encoder: Optional[_Encoder] = None self.ended_by: Optional[str] = None self.error: Optional[str] = None @@ -280,19 +289,16 @@ async def end(self, reason: str, grant: Optional[WriteNamespaceGrant]) -> dict: uploader = NamespaceUploader(grant) if grant is not None else self.uploader if uploader is not None: for segment in self.segments: - if not segment["uploaded"]: + if not segment.entry.uploaded: await self._upload(uploader, segment) with contextlib.suppress(OSError): self.hints.directory.rmdir() - first, last = (self.segments[0], self.segments[-1]) if self.segments else ({}, {}) - manifest = {"recording_id": self.id, "scope": self.hints.scope, "format": MP4, "fps": self.hints.fps, - "cursor": self.cursor, "source": "stream" if self.streams else "capture", - "started_at": first.get("started_at"), "ended_at": last.get("ended_at"), - "ended_by": self.ended_by, "segments": [{k: v for k, v in s.items() if k != "file"} - for s in self.segments]} - if self.error and self.ended_by == "error": - manifest["error"] = self.error - return manifest + entries = [s.entry for s in self.segments] + return RecordingManifest( + recording_id=self.id, scope=self.hints.scope, format=MP4, fps=self.hints.fps, cursor=self.cursor, + source="stream" if self.streams else "capture", started_at=entries[0].started_at if entries else None, + ended_at=entries[-1].ended_at if entries else None, ended_by=self.ended_by, segments=entries, + error=self.error if self.ended_by == "error" else None).wire() async def _add_frame(self, item: RecordedFrame) -> None: frame = await self._in_thread(_video_frame, item.image, self.hints.max_edge) @@ -342,29 +348,36 @@ async def _seal(self, encoder: _Encoder) -> None: async def _finish(self, path: Path, started_at: datetime, ended_at: datetime, width: int, height: int, frames: int) -> None: size, digest = await asyncio.to_thread(_digest, path) - segment = {"index": len(self.segments), "path": f"{self.id}/{path.name}", "started_at": _iso(started_at), - "ended_at": _iso(ended_at), "width": width, "height": height, "frames": frames, "bytes": size, - "sha256": digest, "uploaded": False, "file": path} + segment = _Segment(ManifestSegment( + index=len(self.segments), path=f"{self.id}/{path.name}", started_at=_iso(started_at), + ended_at=_iso(ended_at), width=width, height=height, frames=frames, bytes=size, sha256=digest, + uploaded=False), path) self.segments.append(segment) if self.uploader is not None: task = asyncio.create_task(self._upload(self.uploader, segment)) self.uploads.add(task) task.add_done_callback(self.uploads.discard) - async def _upload(self, uploader: NamespaceUploader, segment: dict) -> None: + async def _upload(self, uploader: NamespaceUploader, segment: _Segment) -> None: try: - await uploader.upload(segment["path"], segment["file"]) + await uploader.upload(segment.entry.path, segment.file) except Exception as e: - segment["upload_error"] = f"{getattr(e, 'code', type(e).__name__)}: {e}" + segment.entry.upload_error = f"{getattr(e, 'code', type(e).__name__)}: {e}" return - segment["uploaded"] = True - segment.pop("upload_error", None) - segment["file"].unlink(missing_ok=True) + segment.entry.uploaded, segment.entry.upload_error = True, None + segment.file.unlink(missing_ok=True) async def _in_thread(self, fn, *args): return await asyncio.get_running_loop().run_in_executor(self.executor, fn, *args) +@dataclass +class _Segment: + """A closed segment: its manifest entry, and its file until it uploads.""" + entry: ManifestSegment + file: Path + + class _Encoder: """One segment's H.264 encoder: each frame's timestamp is its ``captured_at`` from the segment's first frame, so the video's timeline is the computer's clock.""" @@ -376,8 +389,8 @@ def __init__(self, path: Path, width: int, height: int, fps: int, started_at: da self.frames = 0 self._last_pts = -1 self.container = av.open(str(path), "w", format="mp4", options={"movflags": "+faststart"}) - self.stream = self.container.add_stream( - "libx264", rate=fps, options={"preset": "ultrafast", "tune": "zerolatency", "crf": "28", "g": str(2 * fps)}) + self.stream = self.container.add_stream("libx264", rate=fps, + options={**H264_OPTIONS, "g": str(KEYFRAME_S * fps)}) self.stream.width, self.stream.height, self.stream.pix_fmt = width, height, "yuv420p" self.stream.codec_context.time_base = Fraction(1, 1000) @@ -396,6 +409,44 @@ def close(self) -> None: self.container.close() +async def frames_in_thread(frames: Callable[[], Iterator[RecordedFrame]], + keep: int = THREAD_QUEUE) -> AsyncIterator[RecordedFrame]: + """A ``@record`` stream from a blocking source: the generator ``frames()`` runs in a thread of its own, the + latest ``keep`` of its frames wait for the encoder, what it raises is raised here, and it is closed when the + recording stops.""" + loop, queue, stop = asyncio.get_running_loop(), asyncio.Queue(), threading.Event() + + def offer(item: Any) -> None: + if queue.qsize() >= keep: + queue.get_nowait() + queue.put_nowait(item) + + def post(item: Any) -> None: + with contextlib.suppress(RuntimeError): # the loop has closed: nobody is reading + loop.call_soon_threadsafe(offer, item) + + def pump() -> None: + try: + with contextlib.closing(frames()) as source: + for item in source: + if stop.is_set(): + return + post(item) + except Exception as e: + post(e) + finally: + post(None) + + threading.Thread(target=pump, name="computer-record", daemon=True).start() + try: + while (item := await queue.get()) is not None: + if isinstance(item, Exception): + raise item + yield item + finally: + stop.set() + + async def _poll(runtime: ComputerRuntime, hints: RecordHints, cursor: bool) -> AsyncIterator[RecordedFrame]: """The fallback stream: ``@capture`` every ``1/fps`` seconds, with a pointer drawn where the driver says it is.""" interval, due = 1 / hints.fps, time.monotonic() diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index 0774556e7..b642881b7 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -152,18 +152,25 @@ async def test_each_way_a_recording_can_fall_short_fails_its_check(home, monkeyp monkeypatch.setattr(checks, "RECORD_S", 0.8) real_end = recording_module.Recordings.end - async def miscounted(self, recording_id, upload, reason): - manifest = await real_end(self, recording_id, upload, reason) - manifest["segments"][0]["frames"] += 1 - return manifest - - monkeypatch.setattr(recording_module.Recordings, "end", miscounted) - miscount = await _recording_checks(Recorder()) + async def altered(change): + async def end(self, recording_id, upload, reason): + manifest = await real_end(self, recording_id, upload, reason) + change(manifest) + return manifest + + monkeypatch.setattr(recording_module.Recordings, "end", end) + return await _recording_checks(Recorder()) + + miscount = await altered(lambda m: m["segments"][0].update(frames=m["segments"][0]["frames"] + 1)) + unmeasured = await altered(lambda m: m["segments"][0].pop("sha256")) + malformed = await altered(lambda m: m.pop("ended_by")) assert "s between segments 0 and 1" in gap[NAMES[0]][1] assert "it ended by 'limit'" in limited[NAMES[0]][1] assert "couldn't be read back" in unreadable[NAMES[0]][1] assert "stopped reading back at byte 0" in truncated[NAMES[0]][1] assert "the manifest says 1280x800" in miscount[NAMES[0]][1] + assert "segments [0] lack their times" in unmeasured[NAMES[0]][1] + assert "isn't a computer-recording/v1 manifest" in malformed[NAMES[0]][1] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 5669e6b66..1b9a3fb1c 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -4,6 +4,7 @@ import asyncio import hashlib +import threading import time from datetime import datetime, timedelta, timezone from fractions import Fraction @@ -24,6 +25,7 @@ RecordedFrame, RecordedSegment, Recording, + frames_in_thread, record, ) from agentenv_protocol.computer import recording as recording_module # noqa: E402 @@ -216,7 +218,7 @@ async def frames(self, hints): _, manifest = await _record(Broken(), 2.5) assert manifest["ended_by"] == "error" and manifest["error"] == "RuntimeError: no display" - assert manifest["segments"] == [] and manifest["started_at"] is None + assert manifest["segments"] == [] and "started_at" not in manifest @pytest.mark.asyncio @@ -362,6 +364,7 @@ def add(self, frame, at): @pytest.mark.asyncio async def test_only_the_latest_starts_and_ends_are_kept_for_retries(home, monkeypatch): monkeypatch.setattr(recording_module, "KEEP_ENDED", 1) + monkeypatch.setattr(recording_module, "KEEP_STARTED", 4) driver = Recorded() async with http_client(driver) as http: for name in "abcde": @@ -438,3 +441,41 @@ async def test_recordings_that_stop_themselves_release_their_thread_and_only_the assert len(running) == 2 and sum(not r.done for r in running.values()) == 1 for recording_id in list(running): await http.post(END, json={"recording_id": recording_id}) + + +def _counted(n: int, closed: threading.Event, fail: Exception | None = None): + """A blocking source of ``n`` frames numbered from 0, a millisecond apart, that may fail after them.""" + def frames(): + try: + for i in range(n): + yield RecordedFrame(i, datetime.now(timezone.utc)) + time.sleep(0.001) + if fail is not None: + raise fail + finally: + closed.set() + return frames + + +@pytest.mark.asyncio +async def test_frames_in_thread_hands_over_a_blocking_sources_frames_and_what_it_raises(): + closed = threading.Event() + with pytest.raises(RuntimeError, match="the X server went away"): + got = [f.image async for f in frames_in_thread(_counted(5, closed, RuntimeError("the X server went away")))] + got = [f.image async for f in frames_in_thread(_counted(5, closed))] + assert got == [0, 1, 2, 3, 4] and closed.is_set() + + +@pytest.mark.asyncio +async def test_frames_in_thread_keeps_the_latest_frames_for_a_slow_reader_and_closes_its_source_when_stopped(): + done, stopped = threading.Event(), threading.Event() + stream = frames_in_thread(_counted(30, done), keep=4) + first = (await anext(stream)).image + await asyncio.to_thread(done.wait, 5) + await asyncio.sleep(0.05) + rest = [f.image async for f in stream] + endless = frames_in_thread(_counted(1_000_000, stopped)) + await anext(endless) + await endless.aclose() + assert first == 0 and len(rest) < 4 and rest[-1] == 29 + assert await asyncio.to_thread(stopped.wait, 5) diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index 32286f2e0..385cd852e 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -12,7 +12,7 @@ import time from collections import OrderedDict from datetime import datetime, timezone -from typing import Any, AsyncIterator, Callable +from typing import Any, AsyncIterator, Callable, Iterator import av import mss @@ -37,6 +37,7 @@ computer_driver, display, drag, + frames_in_thread, key_down, key_up, pointer_down, @@ -65,8 +66,6 @@ #: How far apart a drag's pointer moves are, in pixels. DRAG_STEP_PX = 20 DROPPED = (ConnectionClosedError, DisplayConnectionError, OSError) -#: How many recorded frames wait for the encoder before the oldest is dropped. -RECORD_QUEUE = 8 @computer_driver(name="reference", @@ -97,42 +96,19 @@ async def grab(self) -> Image: @record async def stream(self, hints: RecordHints) -> AsyncIterator[RecordedFrame]: - """The display through FFmpeg's ``x11grab``, read in a thread of its own. Its frames carry the wall clock, - so each one's time is when it was captured.""" + """The display through FFmpeg's ``x11grab``. Its frames carry the wall clock, so each one's time is when it + was captured.""" size = await self.screen() - loop, frames, stop = asyncio.get_running_loop(), asyncio.Queue(), threading.Event() + options = {"framerate": str(hints.fps), "video_size": f"{size.width}x{size.height}", + "draw_mouse": "1" if hints.cursor else "0"} - def offer(item: Any) -> None: - if frames.qsize() >= RECORD_QUEUE: - frames.get_nowait() - frames.put_nowait(item) + def frames() -> Iterator[RecordedFrame]: + with av.open(os.environ.get("DISPLAY", ":0"), format="x11grab", options=options) as source: + for frame in source.decode(video=0): + yield RecordedFrame(frame, datetime.fromtimestamp(frame.time, timezone.utc)) - def post(item: Any) -> None: - with contextlib.suppress(RuntimeError): # the loop has closed: nobody is reading - loop.call_soon_threadsafe(offer, item) - - def pump() -> None: - options = {"framerate": str(hints.fps), "video_size": f"{size.width}x{size.height}", - "draw_mouse": "1" if hints.cursor else "0"} - try: - with av.open(os.environ.get("DISPLAY", ":0"), format="x11grab", options=options) as source: - for frame in source.decode(video=0): - if stop.is_set(): - return - post(RecordedFrame(frame, datetime.fromtimestamp(frame.time, timezone.utc))) - except Exception as e: - post(e) - finally: - post(None) - - threading.Thread(target=pump, name="reference-record", daemon=True).start() - try: - while (item := await frames.get()) is not None: - if isinstance(item, Exception): - raise item - yield item - finally: - stop.set() + async for frame in frames_in_thread(frames): + yield frame async def ready_checks(self) -> list[ReadyCheck]: try: diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 15c29cedd..24c5828f6 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -10,6 +10,7 @@ from typing import ClassVar, Optional from agentenv_protocol.computer.card import COMPUTER_RECORDING +from agentenv_protocol.computer.manifest import ManifestSegment, RecordingManifest from agentenv_protocol.client import find_extension from agent_env.a2a_agent.object_transfer import ObjectLimits, namespace_grant @@ -148,8 +149,8 @@ async def end_recording(context: TaskStepContext, name: str, reason: str) -> Non return try: deployed = _deployed(context, entry["env_id"]) - manifest = await deployed.invoke(COMPUTER_RECORDING, "end", { - "recording_id": entry["recording_id"], "upload": _grant(deployed, entry["namespace"]), "reason": reason}) + manifest = RecordingManifest.model_validate(await deployed.invoke(COMPUTER_RECORDING, "end", { + "recording_id": entry["recording_id"], "upload": _grant(deployed, entry["namespace"]), "reason": reason})) except Exception as e: logger.warning("end_computer_recording %s: %s; registering what its namespace holds", name, e) entry["error"] = f"{type(e).__name__}: {e}"[:500] @@ -165,10 +166,10 @@ async def end_recording(context: TaskStepContext, name: str, reason: str) -> Non logger.warning("end_computer_recording %s: registering it failed: %s", name, e) entry["state"], entry["error"] = "failed", f"{type(e).__name__}: {e}"[:500] return - uploaded = [s for s in manifest["segments"] if s.get("uploaded")] - entry.update(state="ended", ended_by=manifest["ended_by"], segments=len(uploaded), - missing_segments=len(manifest["segments"]) - len(uploaded)) - logger.info("end_computer_recording %s: %d segments, ended by %s", name, len(uploaded), manifest["ended_by"]) + uploaded = [s for s in manifest.segments if s.uploaded] + entry.update(state="ended", ended_by=manifest.ended_by, segments=len(uploaded), + missing_segments=len(manifest.segments) - len(uploaded)) + logger.info("end_computer_recording %s: %d segments, ended by %s", name, len(uploaded), manifest.ended_by) def _deployed(context: TaskStepContext, env_id: str): @@ -194,7 +195,7 @@ def _grant(deployed, namespace: str) -> dict: if not issues_grants_to(store, sandbox_type): raise RuntimeError(f"the object store issues no upload grant that reaches env {deployed.env_id}'s sandbox " f"({sandbox_type or 'unknown'}), so its recording couldn't upload") - duration = params.get("max_duration_s", 3600) + duration = params["max_duration_s"] cap = min(store.max_single_upload_bytes or _MAX_SEGMENT_BYTES, _MAX_SEGMENT_BYTES) limits = ObjectLimits(max_objects=MAX_SEGMENTS, max_object_bytes=cap, max_total_bytes=MAX_RECORDING_BYTES) try: @@ -204,16 +205,17 @@ def _grant(deployed, namespace: str) -> dict: return grant.model_dump(mode="json") -def _from_namespace(entry: dict) -> dict: +def _from_namespace(entry: dict) -> RecordingManifest: """A manifest of what the recording's namespace holds, for a recording its driver can't end.""" store = get_config().get_object_store() root = entry["namespace"].rstrip("/") + "/" paths = sorted(url[len(root):] for url in store.list_at(root) if url.endswith(".mp4")) - return {"recording_id": entry.get("recording_id"), "ended_by": "error", "error": entry.get("error"), - "segments": [{"index": i, "path": path, "uploaded": True} for i, path in enumerate(paths)]} + return RecordingManifest(recording_id=entry.get("recording_id"), ended_by="error", error=entry.get("error"), + segments=[ManifestSegment(index=i, path=path, uploaded=True) + for i, path in enumerate(paths)]) -def _register(context: TaskStepContext, name: str, namespace: str, manifest: dict) -> dict: +def _register(context: TaskStepContext, name: str, namespace: str, manifest: RecordingManifest) -> dict: """The uploaded segments and the manifest as FileArtifacts, pointing at the objects, in one universe.""" universe_id = derive_id(context.instance_id or "run", f"recording-{name}") if is_local_id(universe_id): @@ -221,12 +223,12 @@ def _register(context: TaskStepContext, name: str, namespace: str, manifest: dic objects, documents = get_config().get_object_store(), get_artifact_store() root = namespace.rstrip("/") manifest_url = objects.put(objects.get_object_key(f"{root}/manifest.json"), - json.dumps(manifest, indent=1).encode(), content_type="application/json", + json.dumps(manifest.wire(), indent=1).encode(), content_type="application/json", allow_overwrite=True) files = {"manifest.json": (manifest_url, "application/json")} - for segment in manifest["segments"]: - if segment.get("uploaded"): - files[segment["path"].replace("/", "-")] = (f"{root}/{segment['path']}", "video/mp4") + for segment in manifest.segments: + if segment.uploaded: + files[segment.path.replace("/", "-")] = (f"{root}/{segment.path}", "video/mp4") stored = {filename: documents.put_document(FileArtifact( id=derive_id(universe_id, filename), filename=filename, content_type=content_type, object_url=url, description=f"Recording {name!r} of task instance {context.instance_id}")) diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index eb792ce57..3a3873227 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -29,11 +29,12 @@ class _Computer: """A deployed env whose driver records: each end uploads one segment per started recording under its grant, - or leaves the last one out when ``drop_last``.""" + or leaves the last one out when ``drop_last``; it ``forgets`` the recording, or ``garbles`` its manifest.""" - def __init__(self, store, env_id: str = "desktop", forgets: bool = False, drop_last: bool = False) -> None: + def __init__(self, store, env_id: str = "desktop", forgets: bool = False, garbles: bool = False, + drop_last: bool = False) -> None: self.env_id, self.environment_card, self.sandbox_type = env_id, _CARD, "local" - self.store, self.forgets, self.drop_last = store, forgets, drop_last + self.store, self.forgets, self.garbles, self.drop_last = store, forgets, garbles, drop_last self.calls: list[tuple[str, dict]] = [] self.started: set[str] = set() @@ -47,6 +48,8 @@ async def invoke(self, uri, method, params=None, **_): return {"recording_id": "rec-1", "started_at": "2026-10-27T10:00:00.000Z", "format": "video/mp4"} if self.forgets: raise RuntimeError("400 invalid_argument: there is no recording 'rec-1' on this computer") + if self.garbles: + return {"recording_id": "rec-1", "segments": "all of them"} segments = [{"index": 0, "path": "rec-1/0000.mp4", "uploaded": True}, {"index": 1, "path": "rec-1/0001.mp4", "uploaded": not self.drop_last}] if not self.drop_last: @@ -103,10 +106,11 @@ async def test_a_segment_the_driver_couldnt_upload_is_left_out_and_counted(store @pytest.mark.asyncio -async def test_a_recording_the_driver_forgot_is_registered_from_its_namespace(store): - context = await _start().execute(_context(_Computer(store, forgets=True))) +@pytest.mark.parametrize("fault, error", [("forgets", "no recording 'rec-1'"), ("garbles", "ValidationError")]) +async def test_a_recording_the_driver_forgot_or_garbled_is_registered_from_its_namespace(store, fault, error): + context = await _start().execute(_context(_Computer(store, **{fault: True}))) entry = (await _end().execute(context)).metadata[RECORDINGS_KEY]["desk"] - assert entry["state"] == "ended" and entry["ended_by"] == "error" and "no recording 'rec-1'" in entry["error"] + assert entry["state"] == "ended" and entry["ended_by"] == "error" and error in entry["error"] assert sorted(Artifact.get(entry["artifact"]["id"]).file_artifact_ids) == ["manifest.json", "rec-1-0000.mp4"] From 16a275a70515934e10838af8324c832972e87c82 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 08:46:33 -0700 Subject: [PATCH 13/17] test(computer): recording chaos in the integration tier The reference driver's desktop in a container of its own: the X server killed mid-recording, segments rolling over, a full disk, 64 clicks while recording at 30 fps, and the driver restarted, with every segment read back and decoded. Co-Authored-By: Claude Opus 5.5 --- .../computer/recording_chaos_test.py | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tst/integration/computer/recording_chaos_test.py diff --git a/tst/integration/computer/recording_chaos_test.py b/tst/integration/computer/recording_chaos_test.py new file mode 100644 index 000000000..a486f7e2e --- /dev/null +++ b/tst/integration/computer/recording_chaos_test.py @@ -0,0 +1,170 @@ +"""computer-recording/v1 on the scaffold's desktop when things go wrong: the X server killed mid-recording, segments +rolling over, the disk filling up, clicks while it records at 30 fps, and the driver restarted. Each test runs the +scaffold's image in a container of its own, so it can kill, fill and restart it, and decodes every segment it reads +back. Needs a Docker daemon.""" + +import asyncio +import json +import subprocess +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +import av +import pytest + +from agentenv_protocol.client import ExtensionCallError +from agentenv_protocol.computer import COMPUTER_READY, COMPUTER_RECORDING, Recording, RecordingManifest +from agentenv_protocol.computer.conformance.client import Driver +from agentenv_protocol.computer.recording import DISK_CHECK_S, RECORDINGS_DIR +from tst.integration.computer.scaffold_test import _scaffold, _when + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +PORT = 18765 +RECORDINGS = f"/home/user/{RECORDINGS_DIR}" +SEGMENT_S = Recording().max_segment_s +FPS = 10 + + +@pytest.fixture(scope="module") +def image(tmp_path_factory): + tag = f"agentenv-test-recording-desk:{uuid.uuid4().hex[:8]}" + _docker("build", "-q", "-t", tag, str(_scaffold(tmp_path_factory.mktemp("desk") / "scaffold") / "envs" / "desktop")) + yield tag + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) + + +@pytest.mark.asyncio +async def test_a_recording_outlives_its_x_server_with_a_gap_and_a_new_segment(image, tmp_path): + async with _desk(image) as (name, driver): + started = await _start(driver) + await asyncio.sleep(3) + _docker("exec", name, "pkill", "-9", "Xvfb") + await asyncio.sleep(6) + manifest = await _end(driver, started) + decoded = _decoded(name, manifest, tmp_path) + assert manifest.ended_by == "end" and len(manifest.segments) >= 2 + assert max(_gaps(manifest)) >= 0.5 and decoded == [s.frames for s in manifest.segments] + + +@pytest.mark.asyncio +async def test_segments_roll_over_at_max_segment_s_with_no_frame_lost_between_them(image, tmp_path): + async with _desk(image) as (name, driver): + started = await _start(driver) + await asyncio.sleep(2 * SEGMENT_S + 2) + manifest = await _end(driver, started) + decoded = _decoded(name, manifest, tmp_path) + spans = [(_when(s.ended_at) - _when(s.started_at)).total_seconds() for s in manifest.segments] + assert len(spans) == 3 and all(span < SEGMENT_S for span in spans) + assert all(gap < 2 / FPS for gap in _gaps(manifest)) and decoded == [s.frames for s in manifest.segments] + + +@pytest.mark.asyncio +async def test_a_full_disk_ends_a_recording_as_limit_and_what_it_recorded_plays(image, tmp_path): + tmpfs = f"type=tmpfs,destination={RECORDINGS},tmpfs-size={600 << 20},tmpfs-mode=1777" + async with _desk(image, "--mount", tmpfs) as (name, driver): + started = await _start(driver) + await asyncio.sleep(2) + _docker("exec", name, "dd", "if=/dev/zero", f"of={RECORDINGS}/fill", "bs=1M", "count=150") + await asyncio.sleep(2 * DISK_CHECK_S) + manifest = await _end(driver, started) + decoded = _decoded(name, manifest, tmp_path) + assert manifest.ended_by == "limit" and decoded and decoded == [s.frames for s in manifest.segments] + + +@pytest.mark.asyncio +async def test_every_click_lands_while_the_display_records_at_30_fps(image): + async with _desk(image) as (_name, driver): + layout = await _layout(driver) + (gx, gy), cell = layout["grid"][:2], layout["cell"] + started = await _start(driver, fps=30) + async with driver.session() as session: + await session.call_tool("computer_screenshot", {}) + for i in range(64): + x, y = gx + (i % 8) * cell + cell // 2, gy + (i // 8) * cell + cell // 2 + clicked = await session.call_tool("computer_click", {"x": x, "y": y}) + assert not clicked.isError, clicked.content[0].text + await asyncio.sleep(1) + cells = json.loads((await driver.run("cat", "fixture/state.json"))["stdout"])["cells"] + manifest = await _end(driver, started) + span = (_when(manifest.segments[-1].ended_at) - _when(manifest.segments[0].started_at)).total_seconds() + assert len(cells) == 64 and sum(s.frames for s in manifest.segments) / span > 25 + + +@pytest.mark.asyncio +async def test_a_restarted_driver_forgets_its_recording_and_keeps_the_segments_it_closed(image): + async with _desk(image) as (name, driver): + started = await _start(driver) + await asyncio.sleep(SEGMENT_S + 2) + _docker("restart", "-t", "2", name) + driver = await _ready(name) + with pytest.raises(ExtensionCallError) as refused: + await _end(driver, started) + kept = (await driver.run("ls", f"{RECORDINGS}/{started['recording_id']}"))["stdout"].split() + assert refused.value.code == "invalid_argument" and "0000.mp4" in kept + + +@asynccontextmanager +async def _desk(image: str, *docker_args: str): + """The desktop in a container of its own, and a client for its driver once it is ready.""" + name = f"agentenv-test-desk-{uuid.uuid4().hex[:8]}" + _docker("run", "-d", "--name", name, "-p", f"127.0.0.1::{PORT}", *docker_args, + "-e", 'AGENTENV_COMPUTER_SETTINGS={"data_paths": ["fixture"]}', image) + try: + yield name, await _ready(name) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + +async def _ready(name: str) -> Driver: + """A client for the container's driver once its computer is ready; a restart may move its port.""" + for _ in range(240): + port = subprocess.run(["docker", "port", name, str(PORT)], capture_output=True, text=True).stdout + if port: + try: + driver = await Driver.connect(f"http://{port.splitlines()[0]}", 30) + if (await driver.invoke(COMPUTER_READY, "status"))["ready"]: + return driver + except Exception: # not serving yet + pass + await asyncio.sleep(0.25) + raise TimeoutError(f"{name}'s computer never became ready") + + +async def _layout(driver: Driver) -> dict: + for _ in range(120): + out = await driver.run("cat", "fixture/layout.json") + if out["exit_code"] == 0 and out["stdout"].strip(): + return json.loads(out["stdout"]) + await asyncio.sleep(0.25) + raise TimeoutError("the Fixture Desk never wrote its layout") + + +async def _start(driver: Driver, fps: int = FPS) -> dict: + return await driver.invoke(COMPUTER_RECORDING, "start", {"request_id": uuid.uuid4().hex, "fps": fps}) + + +async def _end(driver: Driver, started: dict) -> RecordingManifest: + return RecordingManifest.model_validate( + await driver.invoke(COMPUTER_RECORDING, "end", {"recording_id": started["recording_id"]})) + + +def _decoded(name: str, manifest: RecordingManifest, tmp: Path) -> list[int]: + """How many frames each segment decodes to, read back out of the container.""" + counts = [] + for segment in manifest.segments: + local = tmp / segment.path.replace("/", "-") + local.write_bytes(_docker("exec", name, "cat", f"{RECORDINGS}/{segment.path}").stdout) + with av.open(str(local)) as video: + counts.append(sum(1 for _ in video.decode(video=0))) + return counts + + +def _gaps(manifest: RecordingManifest) -> list[float]: + segments = manifest.segments + return [(_when(b.started_at) - _when(a.ended_at)).total_seconds() for a, b in zip(segments, segments[1:])] + + +def _docker(*args: str) -> subprocess.CompletedProcess: + return subprocess.run(["docker", *args], check=True, capture_output=True) From 8e90e988c8e9df88de16876ed94bacdbbb040952 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 09:07:29 -0700 Subject: [PATCH 14/17] fix(computer): never drop a source's error, and time recording tests from their first frame frames_in_thread held its source's end and error in the same drop-oldest queue as frames, so with a small keep the end could push the error out and the recorder retried without saying why. They are now kept apart from the frames, and a slow reader gets exactly the newest keep frames. The recording tests timed a recording from its start, so on a loaded runner the first frame came late and an expected rollover hadn't happened. They now count from the first frame. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/recording.py | 48 ++++++++++++------- .../tests/test_computer_recording.py | 19 +++++++- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 6ce916e0e..811642259 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -12,7 +12,7 @@ import threading import time import uuid -from collections import OrderedDict +from collections import OrderedDict, deque from collections.abc import AsyncIterator, Callable, Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -412,37 +412,49 @@ def close(self) -> None: async def frames_in_thread(frames: Callable[[], Iterator[RecordedFrame]], keep: int = THREAD_QUEUE) -> AsyncIterator[RecordedFrame]: """A ``@record`` stream from a blocking source: the generator ``frames()`` runs in a thread of its own, the - latest ``keep`` of its frames wait for the encoder, what it raises is raised here, and it is closed when the - recording stops.""" - loop, queue, stop = asyncio.get_running_loop(), asyncio.Queue(), threading.Event() + latest ``keep`` of its frames wait for the encoder, what it raises is raised here after them, and it is closed + when the recording stops.""" + loop, stop, wake = asyncio.get_running_loop(), threading.Event(), asyncio.Event() + held: deque[RecordedFrame] = deque(maxlen=keep) + ended: list[Optional[Exception]] = [] # the source's end, and what it raised: never dropped like a frame - def offer(item: Any) -> None: - if queue.qsize() >= keep: - queue.get_nowait() - queue.put_nowait(item) + def hold(frame: RecordedFrame) -> None: + held.append(frame) + wake.set() - def post(item: Any) -> None: + def end(error: Optional[Exception]) -> None: + ended.append(error) + wake.set() + + def post(fn: Callable, item: Any) -> None: with contextlib.suppress(RuntimeError): # the loop has closed: nobody is reading - loop.call_soon_threadsafe(offer, item) + loop.call_soon_threadsafe(fn, item) def pump() -> None: + error = None try: with contextlib.closing(frames()) as source: - for item in source: + for frame in source: if stop.is_set(): return - post(item) + post(hold, frame) except Exception as e: - post(e) + error = e finally: - post(None) + post(end, error) threading.Thread(target=pump, name="computer-record", daemon=True).start() try: - while (item := await queue.get()) is not None: - if isinstance(item, Exception): - raise item - yield item + while True: + if held: + yield held.popleft() + elif ended: + if ended[0] is not None: + raise ended[0] + return + else: + wake.clear() + await wake.wait() finally: stop.set() diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index 1b9a3fb1c..cc01bcf68 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -93,13 +93,23 @@ def _grant() -> dict: async def _record(driver, seconds: float, start: dict | None = None, end: dict | None = None) -> tuple[dict, dict]: + """A recording that runs ``seconds`` from its first frame, however long that took to come, and then ends.""" async with http_client(driver) as http: started = (await http.post(START, json={"request_id": "r", "fps": 10, **(start or {})})).json() + await _first_frame(driver, started) await asyncio.sleep(seconds) manifest = (await http.post(END, json={"recording_id": started["recording_id"], **(end or {})})).json() return started, manifest +async def _first_frame(driver, started: dict) -> None: + recording = driver._computer.recordings.running[started["recording_id"]] + for _ in range(1000): + if recording.segments or (recording.encoder and recording.encoder.frames) or recording.done: + return + await asyncio.sleep(0.01) + + def _decoded(path) -> list: with av.open(str(path)) as container: return list(container.decode(container.streams.video[0])) @@ -182,6 +192,7 @@ async def test_a_size_change_starts_a_segment_of_the_new_size(home): driver.recording = Recording(max_segment_s=60) async with http_client(driver) as http: started = (await http.post(START, json={"request_id": "r", "fps": 10})).json() + await _first_frame(driver, started) await asyncio.sleep(0.4) driver.screen.resize(640, 400, 1.0) await asyncio.sleep(0.4) @@ -195,6 +206,7 @@ async def test_captures_that_fail_leave_a_gap_and_recording_resumes(home): driver.recording = Recording(max_segment_s=60) async with http_client(driver) as http: started = (await http.post(START, json={"request_id": "r", "fps": 10})).json() + await _first_frame(driver, started) await asyncio.sleep(0.4) driver.fail["capture"] = DeviceUnavailable("the X server went away") await asyncio.sleep(1.6) @@ -464,6 +476,11 @@ async def test_frames_in_thread_hands_over_a_blocking_sources_frames_and_what_it got = [f.image async for f in frames_in_thread(_counted(5, closed, RuntimeError("the X server went away")))] got = [f.image async for f in frames_in_thread(_counted(5, closed))] assert got == [0, 1, 2, 3, 4] and closed.is_set() + late = frames_in_thread(_counted(5, threading.Event(), RuntimeError("gone")), keep=1) + await anext(late) + await asyncio.sleep(0.2) + with pytest.raises(RuntimeError, match="gone"): + [f async for f in late] @pytest.mark.asyncio @@ -477,5 +494,5 @@ async def test_frames_in_thread_keeps_the_latest_frames_for_a_slow_reader_and_cl endless = frames_in_thread(_counted(1_000_000, stopped)) await anext(endless) await endless.aclose() - assert first == 0 and len(rest) < 4 and rest[-1] == 29 + assert first == 0 and rest == [26, 27, 28, 29] assert await asyncio.to_thread(stopped.wait, 5) From 4db534b9eac883e838debb5389f856ff8ffe3b35 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 09:42:35 -0700 Subject: [PATCH 15/17] feat(computer): record every computer by default, with PyAV in the computer extra Every ComputerDriverEnvironment now serves computer-recording/v1: from its @record stream if it has one, else by polling @capture. recording = Recording(...) sets the limits, and recording = None turns it off. PyAV moves from its own recording extra into computer, so a driver needs one extra and recording.py imports it plainly. agent-env depends on the computer extra, so its installs now carry PyAV (about 45 MB, FFmpeg bundled), and THIRD_PARTY_NOTICES lists it. The probe tests record for 0.8 s instead of 2 s, now that every fake driver records, and the rollover chaos test no longer expects an exact number of segments from a fixed sleep. Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 36 +++++++++++++++++++ packages/agentenv-protocol/README.md | 10 +++--- packages/agentenv-protocol/pyproject.toml | 10 +++--- .../src/agentenv_protocol/computer/checks.py | 2 +- .../src/agentenv_protocol/computer/driver.py | 6 ++-- .../agentenv_protocol/computer/recording.py | 13 ++----- .../tests/test_computer_conformance.py | 6 ++-- .../test_computer_conformance_recording.py | 7 ++-- .../tests/test_computer_recording.py | 36 ++++++++----------- pyproject.toml | 2 +- .../drivers/envs/reference/driver.py | 2 +- .../computer/recording_chaos_test.py | 8 +++-- .../computer/scaffold/envs/desktop/Dockerfile | 2 +- uv.lock | 10 +++--- 14 files changed, 87 insertions(+), 63 deletions(-) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 4933ae092..d34f3ba35 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -107,6 +107,14 @@ The runtime dependencies of `agentenv-framework` and its `aws`, `gcp`, `sail`, ` - Source: - [License text 20](#license-text-20) +### av 19.0.2 + +- License: BSD-3-Clause +- Copyright: Copyright retained by original committers. All rights reserved. +- Source: +- Bundled: its binary wheels include FFmpeg's libraries and the codecs they are built with, among them libx264 and libx265, each under its own license +- [License text 116](#license-text-116) + ### backports.tarfile 1.2.0 - License: MIT @@ -15245,3 +15253,31 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ``` + +### License text 116 + +```text +Copyright retained by original committers. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the project nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index fd6ac22ab..6a9d66065 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -316,7 +316,7 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: A computer driver is what runs inside a computer an agent uses through screenshots and input. Write one as a `ComputerDriverEnvironment` subclass from `agentenv_protocol.computer`, which needs the -`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn and Pillow). +`computer` extra (`pip install 'agentenv-framework-protocol[computer]'`: mcp, uvicorn, Pillow and PyAV). The driver marks the methods that capture the screen and send input; the base class serves the computer contract from them, so the tools, results and errors can't drift from it. A complete driver for an X11 desktop is agent-env's reference driver, in its @@ -367,7 +367,7 @@ pixels), at the display's size times its scale. Input operations may return an ` | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | | `@run_in_session` | no; a Linux default | `computer-exec/v1` | | `@write_files` | no; a Linux default | `data/add` | -| `@record` | no, with `recording = Recording()` | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | +| `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | The class is checked when it is defined: a missing required operation, half a group, two methods for one operation that some platform could both match, or features that don't match the @@ -429,9 +429,9 @@ running the driver, its operations or its dependencies. ### Recording the display -A driver that declares `recording` serves `computer-recording/v1`: video of the display, as MP4 -(H.264) segments, for the framework to start and end around the work it wants on record. Encoding -needs PyAV, from the `recording` extra (`pip install 'agentenv-framework-protocol[computer,recording]'`). +Every driver serves `computer-recording/v1`: video of the display, as MP4 (H.264) segments, for the +framework to start and end around the work it wants on record. `recording = Recording(...)` sets its +limits, and `recording = None` turns it off. ```python from agentenv_protocol.computer import RecordedFrame, Recording, record diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 8e12e00a9..13a3d7cfb 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -35,16 +35,14 @@ agent = [ "regex>=2023.12.25", "uvicorn>=0.30", ] -# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP and crops and encodes frames. +# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, and +# records the display as H.264 with PyAV (computer-recording/v1). computer = [ + "av>=17", "mcp>=1.25,<2", "Pillow>=10.4", "uvicorn>=0.30", ] -# A computer driver that records: computer-recording/v1 encodes H.264 with PyAV. -recording = [ - "av>=17", -] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -54,7 +52,7 @@ dev = [ "mcp>=1.25,<2", "fastmcp>=3.4,<4", # the standalone-fastmcp mount path "Pillow>=10.4", # the computer extra's frame crops and encodings - "av>=17", # the recording extra's encoder, and the tests' decoder + "av>=17", # the computer extra's encoder, and the tests' decoder ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py index a9eef98e6..6d0736eb6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -41,6 +41,6 @@ def check_features(cls: type) -> None: problems.append("display_modes and @set_display come together") problems += check_recording(cls.recording) if "record" in declared(cls) and cls.recording is None: - problems.append("@record needs recording = Recording()") + problems.append("@record needs recording, which recording = None turns off") if problems: raise TypeError(f"{cls.__qualname__}: " + "; ".join(problems)) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index df9c76807..f8f28aa6c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -43,9 +43,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen layouts: ClassVar[tuple[str, ...]] = ("us",) #: The platforms the driver supports, as the driver env declares them: ``[{"os": {...}, "arch": [...]}]``. supports: ClassVar[list] = [] - #: ``Recording()`` serves computer-recording/v1: from ``@record`` if the driver has one, else by polling - #: ``@capture``. - recording: ClassVar[Recording | None] = None + #: computer-recording/v1, from ``@record`` if the driver has one, else by polling ``@capture``; ``None`` turns + #: it off. + recording: ClassVar[Recording | None] = Recording() tool_access = True #: What the base class keeps for the driver, built by ``__init__``. _computer: ComputerRuntime | None = None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py index 811642259..fbd248d3b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/recording.py @@ -1,7 +1,6 @@ """Recording a computer's display, computer-recording/v1: the base class runs each recording from the driver's ``@record`` stream, or by polling ``@capture`` where the driver has none, encodes frames as H.264 segments, uploads -each segment as it closes and seals the recording at ``end`` with its manifest. Encoding needs the ``recording`` -extra (PyAV).""" +each segment as it closes and seals the recording at ``end`` with its manifest.""" from __future__ import annotations import asyncio @@ -21,6 +20,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +import av from PIL import Image as PILImage from PIL import ImageDraw from pydantic import ValidationError @@ -30,11 +30,6 @@ from .frames import to_pil from .manifest import ManifestSegment, RecordingManifest -try: - import av -except ImportError: # the recording extra isn't installed: a start says so - av = None - if TYPE_CHECKING: from .runtime import ComputerRuntime @@ -68,7 +63,7 @@ @dataclass(frozen=True) class Recording: - """A driver's recording features, as a class attribute: ``recording = Recording()`` serves computer-recording/v1.""" + """A driver's recording limits, as its ``recording`` class attribute; every driver has the defaults.""" max_fps: int = 15 max_concurrent: int = 2 max_duration_s: int = 86_400 @@ -154,8 +149,6 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O if sum(not r.done for r in self.running.values()) >= features.max_concurrent: raise LimitExceeded(f"{features.max_concurrent} recordings are running, the most this computer " "records at once; end one first") - if av is None: - raise Unsupported("recording needs PyAV: install agentenv-framework-protocol[computer,recording]") recording_id = f"rec-{uuid.uuid4().hex[:12]}" root = Path.home() / RECORDINGS_DIR hints = dataclasses.replace(hints, directory=root / recording_id) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py index 3cded2072..6b57262e1 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -39,6 +39,7 @@ @pytest.fixture(autouse=True) def home(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(checks, "RECORD_S", 0.8) return tmp_path @@ -61,11 +62,12 @@ async def test_the_fake_driver_passes_every_check_at_full_and_cleans_up(home): report = await _probe(driver) assert report.passed, _failed(report) assert report.driver == "FixtureDriver" and report.level == "full" - assert set(report.extensions.values()) == {"verified"} and len(report.extensions) == 11 + assert set(report.extensions.values()) == {"verified"} and len(report.extensions) == 12 counts = report.counts() assert counts["failed"] == counts["unverified"] == 0 and counts["not_applicable"] == 1 assert {c.check for c in report.checks} == {f"P{n}" for n in range(10)} | {"X"} - assert not (home / ".agentenv-probe").exists() and ["kill", "4242"] in driver.commands + assert not (home / ".agentenv-probe").exists() and not (home / ".agentenv-recordings").exists() + assert ["kill", "4242"] in driver.commands @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py index b642881b7..fbcf36228 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_recording.py @@ -79,8 +79,11 @@ async def test_a_limit_the_driver_doesnt_keep_fails_and_its_recordings_are_still @pytest.mark.asyncio -async def test_a_driver_without_recording_gets_no_recording_checks(home): - assert await _recording_checks(FixtureDriver()) == {} +async def test_a_driver_that_turns_recording_off_gets_no_recording_checks(home): + class Off(FixtureDriver): + recording = None + + assert await _recording_checks(Off()) == {} def _box(kind: bytes, payload: bytes) -> bytes: diff --git a/packages/agentenv-protocol/tests/test_computer_recording.py b/packages/agentenv-protocol/tests/test_computer_recording.py index cc01bcf68..c238dd9bf 100644 --- a/packages/agentenv-protocol/tests/test_computer_recording.py +++ b/packages/agentenv-protocol/tests/test_computer_recording.py @@ -120,25 +120,30 @@ def _at(iso: str) -> datetime: @pytest.mark.asyncio -async def test_a_driver_with_recording_serves_it_and_one_without_does_not(): - async with http_client(Recorded()) as http: - card = (await http.get("/.well-known/agent-env.json")).json() +async def test_every_driver_serves_recording_unless_it_turns_it_off(): + class Off(FakeDriver): + recording = None + async with http_client(FakeDriver()) as http: - plain = (await http.get("/.well-known/agent-env.json")).json() + card = (await http.get("/.well-known/agent-env.json")).json() + async with http_client(Off()) as http: + off = (await http.get("/.well-known/agent-env.json")).json() extension = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) assert set(extension["params"]["methods"]) == {"start", "end"} assert {k: v for k, v in extension["params"].items() if k not in ("methods", "endpoint")} == { "formats": ["video/mp4; codecs=avc1"], "scopes": ["display"], "fps": {"default": 10, "max": 15}, - "cursor": True, "source": "capture", "max_segment_s": 1, "max_concurrent": 2, "max_duration_s": 86_400} - assert COMPUTER_RECORDING not in [e["uri"] for e in plain["capabilities"]["extensions"]] + "cursor": True, "source": "capture", "max_segment_s": 15, "max_concurrent": 2, "max_duration_s": 86_400} + assert COMPUTER_RECORDING not in [e["uri"] for e in off["capabilities"]["extensions"]] def test_recording_features_and_record_are_checked_when_the_class_is_defined(): with pytest.raises(TypeError, match="recording.max_fps is an integer from 1 to 60"): class TooFast(FakeDriver): recording = Recording(max_fps=120) - with pytest.raises(TypeError, match="@record needs recording = Recording()"): - class Undeclared(FakeDriver): + with pytest.raises(TypeError, match="@record needs recording, which recording = None turns off"): + class Off(FakeDriver): + recording = None + @record async def frames(self, hints): yield None @@ -167,10 +172,7 @@ async def test_the_fallback_polls_capture_and_draws_the_pointer(home): @pytest.mark.asyncio async def test_a_driver_without_the_pointer_group_records_without_a_cursor(home): - class Plain(MinimalDriver): - recording = Recording() - - _, manifest = await _record(Plain(), 0.4) + _, manifest = await _record(MinimalDriver(), 0.4) assert manifest["cursor"] is False and manifest["segments"][0]["width"] == 640 @@ -221,8 +223,6 @@ async def test_a_stream_that_keeps_failing_ends_the_recording_with_its_error(hom monkeypatch.setattr(recording_module, "FAILING_S", 0.5) class Broken(FakeDriver): - recording = Recording() - @record async def frames(self, hints): raise RuntimeError("no display") @@ -339,14 +339,6 @@ async def test_bad_calls_are_refused_with_a_code(path, body, status, code): assert refused.status_code == status and refused.json()["error"]["code"] == code -@pytest.mark.asyncio -async def test_without_pyav_a_start_is_refused_naming_the_extra(monkeypatch): - monkeypatch.setattr(recording_module, "av", None) - async with http_client(Recorded()) as http: - refused = await http.post(START, json={"request_id": "r"}) - assert refused.status_code == 501 and "agentenv-framework-protocol[computer,recording]" in refused.text - - @pytest.mark.asyncio async def test_uploaded_segments_are_not_sent_again_at_end(home): upload = AsyncMock() diff --git a/pyproject.toml b/pyproject.toml index 4cd2f1e78..92dfa80c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ dev = [ "agentenv-framework[aws,boxd,daytona,gcp,sail,tensorlake,tenki,vercel]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs - "av>=17", # the protocol's recording extra, whose tests the unit tier runs and which the recording test decodes with + "av>=17", # the protocol's computer extra, whose tests the unit tier runs and which the recording tests decode with "griffelib==2.3.0", # the plugin API check; exact, since what it reports as a break changes between releases "tomli-w==1.2.0", # the Codex agent's config writer, which only its image installs; its tests import it "moto>=5.0.0", diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index 385cd852e..a8de1b3b0 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -1,7 +1,7 @@ """The reference computer driver, for an X11 desktop: mss captures the screen and XTEST, through python-xlib, sends the input. X calls run off the event loop, one at a time, on a connection checked before each call and reopened when it drops, so the driver outlives a restarted X server. It records the display through FFmpeg's ``x11grab``, by PyAV. It -needs ``agentenv-framework-protocol[computer,recording]``, ``mss`` and ``python-xlib`` where it runs.""" +needs ``agentenv-framework-protocol[computer]``, ``mss`` and ``python-xlib`` where it runs.""" from __future__ import annotations import asyncio diff --git a/tst/integration/computer/recording_chaos_test.py b/tst/integration/computer/recording_chaos_test.py index a486f7e2e..09f303e92 100644 --- a/tst/integration/computer/recording_chaos_test.py +++ b/tst/integration/computer/recording_chaos_test.py @@ -52,12 +52,14 @@ async def test_a_recording_outlives_its_x_server_with_a_gap_and_a_new_segment(im async def test_segments_roll_over_at_max_segment_s_with_no_frame_lost_between_them(image, tmp_path): async with _desk(image) as (name, driver): started = await _start(driver) - await asyncio.sleep(2 * SEGMENT_S + 2) + await asyncio.sleep(2 * SEGMENT_S + 5) manifest = await _end(driver, started) decoded = _decoded(name, manifest, tmp_path) spans = [(_when(s.ended_at) - _when(s.started_at)).total_seconds() for s in manifest.segments] - assert len(spans) == 3 and all(span < SEGMENT_S for span in spans) - assert all(gap < 2 / FPS for gap in _gaps(manifest)) and decoded == [s.frames for s in manifest.segments] + gaps = _gaps(manifest) + assert len(spans) >= 3 and all(SEGMENT_S - 2 / FPS <= span < SEGMENT_S for span in spans[:2]), spans + assert all(gap < 2 / FPS for gap in gaps), gaps + assert decoded == [s.frames for s in manifest.segments] @pytest.mark.asyncio diff --git a/tst/integration/computer/scaffold/envs/desktop/Dockerfile b/tst/integration/computer/scaffold/envs/desktop/Dockerfile index 724514c88..b2a3c9140 100644 --- a/tst/integration/computer/scaffold/envs/desktop/Dockerfile +++ b/tst/integration/computer/scaffold/envs/desktop/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY protocol /opt/protocol RUN python3 -m venv /opt/driver \ - && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer,recording]" "mss>=9,<11" "python-xlib>=0.33,<0.34" + && /opt/driver/bin/pip install --no-cache-dir "/opt/protocol[computer]" "mss>=9,<11" "python-xlib>=0.33,<0.34" RUN useradd --create-home --uid 10001 user COPY start.sh driver.py fixture.py /opt/desktop/ USER user diff --git a/uv.lock b/uv.lock index 24f51ffac..96c0b7885 100644 --- a/uv.lock +++ b/uv.lock @@ -208,6 +208,8 @@ agent = [ { name = "uvicorn" }, ] computer = [ + { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "mcp" }, { name = "pillow" }, { name = "uvicorn" }, @@ -224,17 +226,13 @@ dev = [ { name = "python-multipart" }, { name = "regex" }, ] -recording = [ - { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, -] [package.metadata] requires-dist = [ { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'agent'", specifier = ">=0.3.26,<0.4" }, { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'dev'", specifier = ">=0.3.26,<0.4" }, + { name = "av", marker = "extra == 'computer'", specifier = ">=17" }, { name = "av", marker = "extra == 'dev'", specifier = ">=17" }, - { name = "av", marker = "extra == 'recording'", specifier = ">=17" }, { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, @@ -253,7 +251,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "computer", "recording", "dev"] +provides-extras = ["agent", "computer", "dev"] [[package]] name = "aiofile" From d09c51ae479eb7b3d892fa4b478d94d4270c4789 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 10:22:39 -0700 Subject: [PATCH 16/17] refactor(task): one before_teardown for what must reach the stores before sandboxes go teardown_run and the teardown_sandboxes step each drained staged changelogs and ended open recordings in their own way. before_teardown now does both, for the agents and envs being torn down or for all of them, each part within the caller's limit, and the recording steps have one caller. reset_env no longer ends recordings: no computer driver resets its display or restarts on a reset, so a recording carries on across one. Ending them moves to the computer env's close when that env type lands. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/task/teardown.py | 25 +++++----------- .../task_steps/computer_recording.py | 9 +++--- .../task_step/task_steps/reset_env.py | 2 -- .../task_steps/teardown_sandboxes.py | 20 ++++++++++--- tst/unit/task_step/computer_recording_test.py | 29 ++++++++++++++----- 6 files changed, 50 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2ea61018d..031d8be5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed; its deploy is refused until agent-env creates computers). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, whose open recordings `teardown_run`, `teardown_sandboxes` and `reset_env` end first, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, whose open recordings `teardown_run` and `teardown_sandboxes` end first, and more). `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `daytona` (the `daytona` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/task/teardown.py b/src/agent_env/task/teardown.py index b5fe16062..f5fffb014 100644 --- a/src/agent_env/task/teardown.py +++ b/src/agent_env/task/teardown.py @@ -2,9 +2,10 @@ ``teardown_run`` terminates every sandbox a run's context records: its bare sandboxes', its envs' and its agents'. It removes compute only; the instance, its outputs and the artifacts it collected stay. A local -sandbox's work folder goes too, once the sandbox is down. Recordings still open are ended first. ``agent-env run`` and ``agent-env -eval run`` call it after each run. ``Task.run`` never does, so a caller that keeps a run's environment up, as the hub and -the worker do, owns its teardown. +sandbox's work folder goes too, once the sandbox is down. Before any goes, what has to reach the stores does +(``before_teardown``): the changelogs staged on its agents and the recordings open on its envs. ``agent-env run`` +and ``agent-env eval run`` call it after each run. ``Task.run`` never does, so a caller that keeps a run's +environment up, as the hub and the worker do, owns its teardown. """ from __future__ import annotations @@ -14,15 +15,13 @@ from dataclasses import dataclass from typing import NamedTuple, Optional -from agent_env.a2a_agent.staging import drain_all, staged_changelogs from agent_env.env.env import DeployedEnv from agent_env.providers.sandbox_providers.local_sandbox import remove_local_work_dir from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider from agent_env.task.interrupts import tearing_down from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_steps.computer_recording import end_open_recordings from agent_env.task_step.task_steps.teardown_sandboxes import ( - _DEFAULT_PROVIDERS, TORN_DOWN_KEY, _env_sandbox_ids, _sandbox_ids_of, + _DEFAULT_PROVIDERS, TORN_DOWN_KEY, _env_sandbox_ids, _sandbox_ids_of, before_teardown, ) logger = logging.getLogger(__name__) @@ -98,18 +97,8 @@ async def teardown_run(context: TaskStepContext, *, timeout: float = TERMINATE_T tearing_down() pending = _pending(context) terminated, failed = [], [] - try: # a changelog staged on an agent goes into the store before the agent does - async with asyncio.timeout(timeout): - await drain_all(staged_changelogs(context.metadata)) - except TimeoutError: - logger.warning("teardown: staged changelog increments were not all drained within %gs", timeout) - except asyncio.CancelledError: - return TeardownReport(left=pending) - try: # a recording still open is ended, its last segment uploaded and registered, before its computer goes - async with asyncio.timeout(timeout): - await end_open_recordings(context, "teardown") - except TimeoutError: - logger.warning("teardown: open recordings were not all ended within %gs", timeout) + try: + await before_teardown(context, timeout) except asyncio.CancelledError: return TeardownReport(left=pending) diff --git a/src/agent_env/task_step/task_steps/computer_recording.py b/src/agent_env/task_step/task_steps/computer_recording.py index 24c5828f6..ffc9a850a 100644 --- a/src/agent_env/task_step/task_steps/computer_recording.py +++ b/src/agent_env/task_step/task_steps/computer_recording.py @@ -1,7 +1,7 @@ """Record a computer env's display. ``start_computer_recording`` starts a recording on a deployed env whose card declares computer-recording/v1, with a grant to upload its segments under a namespace of its own; ``end_computer_recording`` ends it and registers its segments and manifest as one FileArtifactUniverse. Teardown -and ``reset_env`` end whatever a run left open (``end_open_recordings``).""" +ends whatever a run left open (``end_open_recordings``).""" from __future__ import annotations import json @@ -130,12 +130,13 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: return context -async def end_open_recordings(context: TaskStepContext, reason: str, env_ids: Optional[list[str]] = None) -> None: - """End and register every recording still open in ``context`` (of ``env_ids`` only, when given).""" +async def end_open_recordings(context: TaskStepContext, env_ids: Optional[list[str]] = None) -> None: + """End and register every recording still open in ``context`` (of ``env_ids`` only, when given), before its + computer is torn down.""" for name, entry in list((context.metadata.get(RECORDINGS_KEY) or {}).items()): if entry.get("state") == "open" and (env_ids is None or entry["env_id"] in env_ids): try: - await end_recording(context, name, reason) + await end_recording(context, name, "teardown") except Exception as e: # what tears the computer down goes on regardless entry["state"], entry["error"] = "failed", f"{type(e).__name__}: {e}"[:500] logger.warning("ending recording %s: %s", name, e) diff --git a/src/agent_env/task_step/task_steps/reset_env.py b/src/agent_env/task_step/task_steps/reset_env.py index 15fb14eb6..88059fdde 100644 --- a/src/agent_env/task_step/task_steps/reset_env.py +++ b/src/agent_env/task_step/task_steps/reset_env.py @@ -11,7 +11,6 @@ from typing import Optional from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_steps.computer_recording import end_open_recordings from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep @@ -58,7 +57,6 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if deployed is None: logger.warning("reset_env: no deployed env '%s' in context; skipping", self.env_id) return context - await end_open_recordings(context, "reset", env_ids=[self.env_id]) try: env = Env.get(deployed.env_id, deployed.env_version) await env.reset(deployed) diff --git a/src/agent_env/task_step/task_steps/teardown_sandboxes.py b/src/agent_env/task_step/task_steps/teardown_sandboxes.py index e122e064a..d299a86c9 100644 --- a/src/agent_env/task_step/task_steps/teardown_sandboxes.py +++ b/src/agent_env/task_step/task_steps/teardown_sandboxes.py @@ -35,6 +35,7 @@ TORN_DOWN_KEY = "torn_down_sandbox_ids" + class TeardownSandboxesTaskStep(TaskStep): """Terminate every sandbox behind the named deployed agents, envs and bare sandboxes.""" @@ -88,10 +89,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning(f"teardown_sandboxes: not deployed in this run, skipping: {', '.join(missing)}") already = set(context.metadata.get(TORN_DOWN_KEY) or []) pending = {sid: (stype, default) for sid, (stype, default) in targets.items() if sid not in already} - if self.env_ids: - await end_open_recordings(context, "teardown", env_ids=self.env_ids) - for name in self.agent_names: # a changelog staged on an agent goes into the store before the agent does - await drain_all(staged_changelogs(context.metadata, agent_name=name), within=LAST_DRAIN_SECONDS) + await before_teardown(context, LAST_DRAIN_SECONDS, env_ids=self.env_ids, agent_names=self.agent_names) results = await asyncio.gather( *(_terminate(sid, stype, default) for sid, (stype, default) in pending.items()), return_exceptions=True, @@ -139,6 +137,20 @@ def _resolve( return targets, missing +async def before_teardown(context: TaskStepContext, within: float, env_ids: Optional[list[str]] = None, + agent_names: Optional[list[str]] = None) -> None: + """Save what has to reach the stores before sandboxes go, each part within ``within`` seconds: the changelogs + staged on ``agent_names`` and the recordings open on ``env_ids``, or on every agent and env when None.""" + for name in [None] if agent_names is None else agent_names: + await drain_all(staged_changelogs(context.metadata, agent_name=name), within=within) + if env_ids is None or env_ids: + try: + async with asyncio.timeout(within): + await end_open_recordings(context, env_ids) + except TimeoutError: + logger.warning("open recordings were not all ended within %gs", within) + + def _env_sandbox_ids(env) -> list[tuple[str, Optional[str]]]: """Every (sandbox_id, sandbox_type) of a deployed env: its primary sandbox plus ``sandbox_ids``. diff --git a/tst/unit/task_step/computer_recording_test.py b/tst/unit/task_step/computer_recording_test.py index 3a3873227..978ad99b2 100644 --- a/tst/unit/task_step/computer_recording_test.py +++ b/tst/unit/task_step/computer_recording_test.py @@ -1,9 +1,11 @@ """start_computer_recording and end_computer_recording against a deployed env standing in for a driver: the grant it's handed, the artifact its uploaded segments and manifest become, what's registered when the driver forgot the -recording, and teardown and reset ending what a run left open.""" +recording, teardown ending what a run left open, and a reset leaving it going.""" from __future__ import annotations +import asyncio + import pytest from agentenv_protocol.computer.card import COMPUTER_RECORDING @@ -29,12 +31,13 @@ class _Computer: """A deployed env whose driver records: each end uploads one segment per started recording under its grant, - or leaves the last one out when ``drop_last``; it ``forgets`` the recording, or ``garbles`` its manifest.""" + or leaves the last one out when ``drop_last``; it ``forgets`` the recording, ``garbles`` its manifest, or + ``hangs`` at end.""" def __init__(self, store, env_id: str = "desktop", forgets: bool = False, garbles: bool = False, - drop_last: bool = False) -> None: + hangs: bool = False, drop_last: bool = False) -> None: self.env_id, self.environment_card, self.sandbox_type = env_id, _CARD, "local" - self.store, self.forgets, self.garbles, self.drop_last = store, forgets, garbles, drop_last + self.store, self.forgets, self.garbles, self.hangs, self.drop_last = store, forgets, garbles, hangs, drop_last self.calls: list[tuple[str, dict]] = [] self.started: set[str] = set() @@ -50,6 +53,8 @@ async def invoke(self, uri, method, params=None, **_): raise RuntimeError("400 invalid_argument: there is no recording 'rec-1' on this computer") if self.garbles: return {"recording_id": "rec-1", "segments": "all of them"} + if self.hangs: + await asyncio.sleep(3600) segments = [{"index": 0, "path": "rec-1/0000.mp4", "uploaded": True}, {"index": 1, "path": "rec-1/0001.mp4", "uploaded": not self.drop_last}] if not self.drop_last: @@ -149,18 +154,26 @@ def broken(*args): @pytest.mark.asyncio -async def test_teardown_and_reset_end_what_a_run_left_open_with_their_reason(store): +async def test_a_reset_leaves_a_recording_going_and_teardown_ends_what_a_run_left_open(store): desktop, other = _Computer(store), _Computer(store, env_id="other") context = _context(desktop, other) context = await _start("a").execute(context) context = await _start("b", env_id="other").execute(context) await ResetEnvTaskStep(id="reset", version=None, env_id="other").execute(context) - await teardown_run(context) recordings = context.metadata[RECORDINGS_KEY] - assert (recordings["a"]["ended_by"], recordings["b"]["ended_by"]) == ("teardown", "reset") + assert recordings["b"]["state"] == "open" and [method for method, _ in other.calls] == ["start"] + await teardown_run(context) + assert (recordings["a"]["ended_by"], recordings["b"]["ended_by"]) == ("teardown", "teardown") assert [method for method, _ in desktop.calls] == ["start", "end"] +@pytest.mark.asyncio +async def test_teardown_goes_on_when_a_recording_wont_end_in_time(store): + context = await _start().execute(_context(_Computer(store, hangs=True))) + report = await teardown_run(context, timeout=0.2) + assert context.metadata[RECORDINGS_KEY]["desk"]["state"] == "open" and not report.left + + @pytest.mark.asyncio async def test_a_teardown_sandboxes_step_ends_the_recordings_of_its_envs_only(store): desktop, other = _Computer(store), _Computer(store, env_id="other") @@ -257,6 +270,6 @@ async def broken(context, name, reason): await real(context, name, reason) monkeypatch.setattr(recording_steps, "end_recording", broken) - await recording_steps.end_open_recordings(context, "teardown") + await recording_steps.end_open_recordings(context) recordings = context.metadata[RECORDINGS_KEY] assert (recordings["a"]["state"], recordings["b"]["state"]) == ("failed", "ended") From 155bf3211ea07ded1bc1151b31eb3f1a17efda42 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 11:01:23 -0700 Subject: [PATCH 17/17] test(computer): allow a full segment to read exactly max_segment_s Manifest times are in milliseconds, so a segment a hair under 15 s can read as 15.000; CI saw spans [14.9, 15.0, 4.8]. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/recording_chaos_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tst/integration/computer/recording_chaos_test.py b/tst/integration/computer/recording_chaos_test.py index 09f303e92..b92b45b54 100644 --- a/tst/integration/computer/recording_chaos_test.py +++ b/tst/integration/computer/recording_chaos_test.py @@ -57,7 +57,7 @@ async def test_segments_roll_over_at_max_segment_s_with_no_frame_lost_between_th decoded = _decoded(name, manifest, tmp_path) spans = [(_when(s.ended_at) - _when(s.started_at)).total_seconds() for s in manifest.segments] gaps = _gaps(manifest) - assert len(spans) >= 3 and all(SEGMENT_S - 2 / FPS <= span < SEGMENT_S for span in spans[:2]), spans + assert len(spans) >= 3 and all(SEGMENT_S - 2 / FPS <= span <= SEGMENT_S for span in spans[:2]), spans assert all(gap < 2 / FPS for gap in gaps), gaps assert decoded == [s.frames for s in manifest.segments]