From 7dcdf21cce9aac8b80d35c8b4e49552925fdd6fc Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:58:14 +0800 Subject: [PATCH] [Feat] Opt-in screenshot input for hosted Jev (S1A_JEV_IMAGES, TypeSafe danger zone) Closes #97 --- CHANGELOG.md | 4 + docs/capabilities.md | 2 +- docs/configuration.md | 1 + docs/decision-models.md | 2 +- s1a/agents/desktop.py | 7 +- s1a/decision_models/base.py | 8 + s1a/decision_models/fakes.py | 5 +- s1a/decision_models/jev.py | 137 ++++++++++++++- s1a/decision_models/types.py | 2 +- s1a/decision_models/wire.py | 46 ++++- tests/test_decision_models_jev_images.py | 206 +++++++++++++++++++++++ 11 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 tests/test_decision_models_jev_images.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d5bd633..5220940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ### Added +- Experimental (TypeSafe danger zone): `S1A_JEV_IMAGES=1` lets `--model jev` on the direct TypeSafe backend read one + current screenshot (TypeSafe `assets` plus the `TypeSafe-Danger-Zone-Images` header), resized or rejected to the + documented limits before sending, and enables `--pixel-target` for `--model jev`. Off by default; the proxy + backend refuses it. Step records gain `sent_image`. No live comparison is recorded yet (#97). - `s1a probe --order reverse|shuffle:N` (with `--seed` and `--json PATH`): the option-order check. Every case is asked in file order and again with the same keys and texts in another dict order; the probe reports the flip rate, accuracy in each order and a row per case with every order asked. Verdict and exit code are unchanged diff --git a/docs/capabilities.md b/docs/capabilities.md index 00c4ad4..0bb0217 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -10,7 +10,7 @@ For issue trackers and serving support, see [System1-Omni model tracker #83](htt | Model / checkpoint | Input modality | Question types implemented | Scenario / front | Serving path | Hardware validation in this repository | |---|---|---|---|---|---| -| TypeSafe Jev (`--model jev`; actual served model comes from the configured endpoint) | Text state and enumerated options | `choice`, `noul`; service protocol also defines `score`, but Agents has no corresponding question type | Tool decisions, browser decisions, rails, `decide`/`probe` and MCP | HTTP through TypeSafe or configured OpenRouter-compatible endpoint; configured compatible endpoint can use System1-Omni `/v1/systemone` | The adapter can address configured Jev-compatible services; exact model identity is endpoint-reported. Historical artifacts include Open-Jev-9B and an Open-Jev-27B-v1.1 load attempt; those are endpoint configurations, not model names guaranteed by the `jev` adapter. See below. Other combinations **unverified here.** | +| TypeSafe Jev (`--model jev`; actual served model comes from the configured endpoint) | Text state and enumerated options; with `S1A_JEV_IMAGES=1` on direct TypeSafe, also one current screenshot (**experimental, TypeSafe danger zone**: offline fake-transport tests only, no recorded live run or quality result, #97) | `choice`, `noul`; service protocol also defines `score`, but Agents has no corresponding question type | Tool decisions, browser decisions, rails, `decide`/`probe` and MCP | HTTP through TypeSafe or configured OpenRouter-compatible endpoint; configured compatible endpoint can use System1-Omni `/v1/systemone` | The adapter can address configured Jev-compatible services; exact model identity is endpoint-reported. Historical artifacts include Open-Jev-9B and an Open-Jev-27B-v1.1 load attempt; those are endpoint configurations, not model names guaranteed by the `jev` adapter. See below. Other combinations **unverified here.** | | JEV-27B-VL (`--model jev-vl-served`; `autotrust/JEV-27B-VL`) | Exactly one current PNG/JPEG plus text state and question context | `choice`, `noul` | Browser front and direct `build_model()` factory path; other fronts are not registered | HTTP native flat `/v1/systemone` through the System1-Omni frontend at `JEV_VL_URL` | Local adapter/registration tests use mocks. Single-H800 GPU checks and bounded R3 browser task completion with same-trial video are validated; other tasks and hardware remain unverified. See [validation scope and limits](decision-models.md#served-jev-vl). | | CLM-v0.1-8B (`--model clm`; default serving label `clm-latest`) | Text | `choice`, `noul` | Tool decisions, browser, rails, `decide`/`probe`, MCP | HTTP `/v1/systemone` to a compatible CLM service; real encoder is external | Adapter fixtures include stub responses. A historical ticket-router report records real Qwen3-8B through upstream `clm-serve` on an RTX 4090, but its Agents source SHA is not pinned in the report and the serving engine is not Omni native CLM. It reports 18/90 correct (20%) for the normal tool-loop framing and a 4/12 fit-probe verdict of “not a decision-model task”; the 90 decisions repeat the same 30-ticket set and are not 90 independent tickets. Treat this as poor, provenance-incomplete historical evidence, not current or broad validation. See [CLM report](../evals/ticket_router/CLM.md). | | LAYA (`--model laya`; default `convaiinnovations/laya`, English; `LAYA_SUBFOLDER=multilingual` selects multilingual) | Text | `choice`, `noul` | Tool, browser, rails, `decide`/`probe`; browser state/question shaping is implemented | In-process Python `laya` | Historical English and multilingual CPU/MPS runs are documented below. Other/typed/custom checkpoints and CUDA are unverified here. | diff --git a/docs/configuration.md b/docs/configuration.md index 37c2af9..d3dfb90 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,6 +11,7 @@ Variables can be exported in your shell or placed in a `.env` file at the root o | `TYPESAFE_API_KEY` | `jev` model | *(unset)* | API key for direct TypeSafe decisions endpoint (`https://api.typesafe.ai/v1/systemone`). | | `TYPESAFE_API_URL` | `jev` model | `https://openrouter.ai/api/alpha/decisions` | Endpoint URL for decisions; defaults to OpenRouter proxy, or can be overridden to a custom proxy URL. | | `TYPESAFE_MODEL` | `jev` model | `jev-1.13.0` direct, `typesafe/jev-1.13` through OpenRouter | Jev model id on either backend. Both defaults are pinned versions: no path sends the moving `jev-latest` or `jev-preview` aliases unless you set them here. On the direct path (`TYPESAFE_API_KEY` without `TYPESAFE_API_URL`) it takes a TypeSafe name such as `jev-1.14.0`; an OpenRouter id like `typesafe/jev-1.13` there is a config error. TypeSafe keeps `jev-1.13.0` available and promises 60 days' notice before deprecating it. Step records keep `model` (the id the response reports, else the one sent) and `requested_model` (the one sent). | +| `S1A_JEV_IMAGES` | `jev` model (direct TypeSafe only) | *(unset: off)* | **Experimental (TypeSafe danger zone).** `1`/`true` lets hosted Jev read the observation's screenshot: the browser front then captures one per step, and the desktop `--pixel-target` path accepts `--model jev`. Each request carries one PNG/JPEG in TypeSafe's `assets` map, referenced once from `state.screenshot` (a fresh UUID reference per request), plus the `TypeSafe-Danger-Zone-Images: true` header. Screenshots whose longest side is over 1,568 px are scaled down (aspect kept, sent as PNG); a side under 280 px after that, a file over 16 MiB or a body over 32 MiB is a config error before anything is sent. Step records add `sent_image` (sent width, height, bytes, format, whether it was resized); image tokens are billed as `input_tokens`. Set with the OpenRouter proxy or `TYPESAFE_API_URL` it is a config error. An image request can take longer than the 5 s default deadline; raise `S1A_DECISION_TIMEOUT_S` if it times out. | | `S1A_DECISION_TIMEOUT_S` | `jev`, `jev-vl-served` models | `5` | Total deadline in seconds for one decision. Jev includes retries; JEV-VL shares it across sequential questions and does not retry HTTP requests. Must be finite and above zero. Raise it for a slower System One server behind `TYPESAFE_API_URL` (e.g. a local OneJev-27B or CLM-8B: `30`). | | `JEV_VL_URL` | `jev-vl-served` model | *(unset, required)* | Frontend base URL serving `autotrust/JEV-27B-VL` through System1-Omni's native flat `/v1/systemone` protocol, e.g. `http://127.0.0.1:8080`. Browser and direct factory paths only; requires exactly one current PNG/JPEG image. | | `OPENROUTER_API_KEY` | `jev` model (proxy), chat model fallback | *(unset)* | OpenRouter API key, used for proxying Jev decisions or as a fallback for `LLM_API_KEY`. | diff --git a/docs/decision-models.md b/docs/decision-models.md index 8cfb47b..13bd5ce 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -58,7 +58,7 @@ shorthands; `warm()` and `close()` open and release the backend. | `--model` | class | `name` | notes | |---|---|---|---| -| `jev` | `JevModel(transport)` | `jev` | the request body every front sent before the layer existed, byte for byte; `from_env` picks TypeSafe or the OpenRouter proxy (see [configuration.md](configuration.md)) | +| `jev` | `JevModel(transport)` | `jev` | the request body every front sent before the layer existed, byte for byte; `from_env` picks TypeSafe or the OpenRouter proxy (see [configuration.md](configuration.md)). Text only unless `S1A_JEV_IMAGES=1` on the direct TypeSafe backend (**experimental, TypeSafe danger zone**, #97): then one current screenshot rides in `assets`, referenced from `state.screenshot`, with the danger-zone header; no quality result is claimed for it yet | | `jev-vl-served` | `ServedJevVLModel(url=, timeout_s=)` | `jev-vl-served` | `autotrust/JEV-27B-VL` through native flat `POST /v1/systemone`; exactly one current PNG/JPEG, choice and noul only; sequential questions share one deadline, with no HTTP retry or answer re-ask. Browser and direct factory paths; local tests use mocks, with separate [bounded R3 GPU/browser validation](#served-jev-vl). | | `clm` | `ClmModel(client, model=)` | `clm` | CLM behind `/v1/systemone`: a frozen Qwen3-8B encoder and two projection heads, scored by an engine that owns everything after the encoder. One POST per decision; `MODEL_SERVICE_CONFIG_ERROR` without `CLM_URL`, `MODEL_CALL_FAILED` when nothing is listening, on an error status, or on a body with no `answers`. Records `url`, the `X-Request-Id` it sent and `served_by` (the served names, read once — by `warm()` on the agent fronts, or on the first decision for `decide`/`probe`); choice and noul ([docs/clm.md](clm.md)) | | `laya` | `LayaModel(agent, model=)` | `laya` | one forward pass per call on a thread; `MODEL_SERVICE_CONFIG_ERROR` when `input_tokens` fills the window (Laya cuts the state silently; see `LAYA_MAX_LEN` and `LAYA_HEAD_MAX_LEN` in [configuration.md](configuration.md)); `ValueError` and `RuntimeError` from the library become `MODEL_CALL_FAILED` | diff --git a/s1a/agents/desktop.py b/s1a/agents/desktop.py index b1ec464..9ca20af 100644 --- a/s1a/agents/desktop.py +++ b/s1a/agents/desktop.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Any, AsyncIterator +from s1a.decision_models.wire import decisions_backend_from_env, jev_images_from_env from s1a.desktop.driver import CuaDriver, DriverError, Snapshot, driver_from_env, opened from s1a.desktop.env import ABSTAIN, WindowEnv, clickable, observable from s1a.spec import Budget, Series, ToolAgentSpec, positive_float, positive_int @@ -119,10 +120,14 @@ def make_series(flags: argparse.Namespace) -> Series: and os.getenv("CUA_S1_VARIANT", "nano") == "4b" and os.getenv("CUA_S1_MODALITY", "multimodal") == "multimodal" ) + if pixel_targets and flags.model == "jev": + # experimental hosted Jev screenshots (#97): a config error on the proxy backend, False when unset + visual_model = jev_images_from_env(decisions_backend_from_env()) if pixel_targets and flags.model not in {"rule", "random"} and not visual_model: raise ValueError( "--pixel-target requires a screenshot model: use --model cua with CUA_S1_VARIANT=4b and " - "CUA_S1_MODALITY=multimodal, or an explicit rule/random baseline" + "CUA_S1_MODALITY=multimodal, --model jev with S1A_JEV_IMAGES=1 on the direct TypeSafe backend " + "(experimental), or an explicit rule/random baseline" ) plan = parse_plan(flags.plan) if flags.plan else () if flags.verify_file and not flags.text: diff --git a/s1a/decision_models/base.py b/s1a/decision_models/base.py index 2cc7ae8..a21a9d3 100644 --- a/s1a/decision_models/base.py +++ b/s1a/decision_models/base.py @@ -37,6 +37,14 @@ async def decide(self, body: Json) -> tuple[Json, int]: ... async def close(self) -> None: ... +class JevImageTransport(JevTransport, Protocol): + """A transport that can carry screenshots (``images`` True): image bodies pass per-request ``headers``.""" + + images: bool + + async def decide(self, body: Json, *, headers: dict[str, str] | None = None) -> tuple[Json, int]: ... + + class DecisionModel(ABC): """A model that reads an observation and a discrete action space and returns a distribution over it.""" diff --git a/s1a/decision_models/fakes.py b/s1a/decision_models/fakes.py index e8775ee..ecd0416 100644 --- a/s1a/decision_models/fakes.py +++ b/s1a/decision_models/fakes.py @@ -16,6 +16,7 @@ class ScriptedTransport: """ model = "typesafe/jev-test" + images = False # set True to stand in for a TypeSafe client with S1A_JEV_IMAGES on def __init__( self, @@ -34,12 +35,14 @@ def __init__( self._usage = usage or {"input_tokens": 0, "output_tokens": 0} self._latency_ms = latency_ms self.bodies: list[Json] = [] + self.headers: list[dict[str, str]] = [] # per call; empty when the call passed none async def warm(self) -> None: return None - async def decide(self, body: Json) -> tuple[Json, int]: + async def decide(self, body: Json, *, headers: dict[str, str] | None = None) -> tuple[Json, int]: self.bodies.append(body) + self.headers.append(dict(headers or {})) if self._error is not None: raise self._error if self._answers: diff --git a/s1a/decision_models/jev.py b/s1a/decision_models/jev.py index 5fe0dff..204f5c8 100644 --- a/s1a/decision_models/jev.py +++ b/s1a/decision_models/jev.py @@ -3,9 +3,43 @@ from __future__ import annotations -from s1a.decision_models.base import DecisionModel, JevTransport -from s1a.decision_models.types import ChoiceQuestion, Json, NoulQuestion, Observation, Question, Reply, Usage -from s1a.decision_models.wire import client_from_env, decisions_backend_from_env, decisions_timeout_from_env +import base64 +import io +import json +import uuid +from typing import cast + +from PIL import Image as PillowImage + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import build_error + +from s1a.decision_models.base import DecisionModel, JevImageTransport, JevTransport +from s1a.decision_models.types import ( + ChoiceQuestion, + Image, + Json, + NoulQuestion, + Observation, + Question, + Reply, + Usage, +) +from s1a.decision_models.wire import ( + JEV_IMAGES_HEADER, + client_from_env, + decisions_backend_from_env, + decisions_timeout_from_env, +) + +# TypeSafe's documented image limits (docs.typesafe.ai/go/images, "dreamer-ready", may change between releases). +# An image outside them is rejected by the API, never resized, so we resize or reject before sending. +JEV_IMAGE_MIN_SIDE = 280 +JEV_IMAGE_MAX_SIDE = 1568 +JEV_IMAGE_MAX_BYTES = 16 * 1024 * 1024 +JEV_MAX_BODY_BYTES = 32 * 1024 * 1024 +SCREENSHOT_KEY = "screenshot" +_FORMATS = {"image/png": ("PNG", "png"), "image/jpeg": ("JPEG", "jpg")} def jev_question(question: Question) -> Json: @@ -30,6 +64,73 @@ def jev_question(question: Question) -> Json: raise TypeError(f"not a question: {question!r}") +def jev_image_asset(image: Image) -> tuple[Json, Json]: + """One screenshot as a TypeSafe asset, plus what was sent (for step records). + + PNG or JPEG still only. The longest side is scaled down to ``JEV_IMAGE_MAX_SIDE`` with the aspect ratio kept + (re-encoded as PNG); an image whose shorter side is then under ``JEV_IMAGE_MIN_SIDE``, or whose file is over + ``JEV_IMAGE_MAX_BYTES``, is a config error, so nothing the API would reject is sent. + """ + if image.media_type not in _FORMATS or not image.data: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"hosted Jev reads PNG or JPEG screenshots, not {image.media_type!r}", + ) + pillow_format, wire_format = _FORMATS[image.media_type] + data = image.data + try: + with PillowImage.open(io.BytesIO(data)) as picture: + if ( + picture.format != pillow_format + or getattr(picture, "n_frames", 1) != 1 + or getattr(picture, "custom_mimetype", None) == "image/apng" + ): + raise ValueError("not a still image matching its media type") + picture.load() + width, height = picture.size + resized = max(width, height) > JEV_IMAGE_MAX_SIDE + if resized: + scale = JEV_IMAGE_MAX_SIDE / max(width, height) + size = (max(1, round(width * scale)), max(1, round(height * scale))) + converted = picture if picture.mode in {"RGB", "RGBA", "L", "LA"} else picture.convert("RGBA") + out = io.BytesIO() + converted.resize(size, PillowImage.Resampling.LANCZOS).save(out, format="PNG") + data, wire_format = out.getvalue(), "png" + width, height = size + except (OSError, ValueError, SyntaxError, PillowImage.DecompressionBombError) as exc: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, cause=exc, error_msg=f"invalid screenshot for hosted Jev: {exc}" + ) from exc + if min(width, height) < JEV_IMAGE_MIN_SIDE: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=( + f"screenshot is {width}x{height} after resizing; hosted Jev rejects a side under " + f"{JEV_IMAGE_MIN_SIDE} px" + ), + ) + if len(data) > JEV_IMAGE_MAX_BYTES: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"screenshot is {len(data)} bytes; hosted Jev takes at most {JEV_IMAGE_MAX_BYTES} per image", + ) + asset = {"type": "image", "format": wire_format, "data_base64": base64.b64encode(data).decode("ascii")} + sent = {"width": width, "height": height, "bytes": len(data), "format": wire_format, "resized": resized} + return asset, sent + + +def _with_screenshot(state: Json | str, reference: str) -> Json: + """The state with the screenshot's reference under ``screenshot`` (once, so every question reads it).""" + if isinstance(state, str): + return {"text": state, SCREENSHOT_KEY: reference} + if SCREENSHOT_KEY in state: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"the observation state already has a {SCREENSHOT_KEY!r} field; cannot attach the screenshot", + ) + return {**state, SCREENSHOT_KEY: reference} + + class JevModel(DecisionModel): """TypeSafe Jev over HTTP; the transport owns the connection, its retries and the round-trip clock.""" @@ -38,6 +139,8 @@ class JevModel(DecisionModel): def __init__(self, transport: JevTransport) -> None: self._transport = transport + # experimental: only a direct TypeSafe client built with S1A_JEV_IMAGES on carries screenshots (#97) + self.supports_images = bool(getattr(transport, "images", False)) @property def model(self) -> str: @@ -49,7 +152,27 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] "state": observation.state, "questions": {name: jev_question(question) for name, question in questions.items()}, } - payload, ms = await self._transport.decide(body) + sent_image: Json | None = None + if self.supports_images and observation.images: + if len(observation.images) != 1: + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=f"hosted Jev takes one current screenshot, got {len(observation.images)}", + ) + asset, sent_image = jev_image_asset(observation.images[0]) + reference = f"screenshot-{uuid.uuid4()}" # fresh per request, so it cannot match any text by accident + body["state"] = _with_screenshot(observation.state, reference) + body["assets"] = {reference: asset} + size = len(json.dumps(body, ensure_ascii=False).encode("utf-8")) + if size > JEV_MAX_BODY_BYTES: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"request is {size} bytes with the screenshot; TypeSafe takes at most {JEV_MAX_BODY_BYTES}", + ) + transport = cast(JevImageTransport, self._transport) # supports_images implies transport.images + payload, ms = await transport.decide(body, headers={JEV_IMAGES_HEADER: "true"}) + else: + payload, ms = await self._transport.decide(body) fields = payload if isinstance(payload, dict) else {} answers = fields.get("answers") answers = answers if isinstance(answers, dict) else {} @@ -58,7 +181,11 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] latency_ms=ms, usage=Usage.from_payload(fields.get("usage")), model=str(fields.get("model") or self.model), - raw={**fields, "requested_model": self.model}, # which Jev was asked for, next to the one that answered + raw={ + **fields, + "requested_model": self.model, # which Jev was asked for, next to the one that answered + **({"sent_image": sent_image} if sent_image is not None else {}), + }, ) async def warm(self) -> None: diff --git a/s1a/decision_models/types.py b/s1a/decision_models/types.py index 20fbc7a..5d98c46 100644 --- a/s1a/decision_models/types.py +++ b/s1a/decision_models/types.py @@ -11,7 +11,7 @@ Json = dict[str, Any] QuestionType = Literal["choice", "noul"] -PROVENANCE_KEYS = ("served_by", "url", "request_id", "server_timing", "requested_model") +PROVENANCE_KEYS = ("served_by", "url", "request_id", "server_timing", "requested_model", "sent_image") @dataclass(frozen=True) diff --git a/s1a/decision_models/wire.py b/s1a/decision_models/wire.py index aa396e9..59bbdee 100644 --- a/s1a/decision_models/wire.py +++ b/s1a/decision_models/wire.py @@ -32,6 +32,10 @@ # double-billed decision when the server closed the socket after reading the request instead. _RETRIED_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError) _MAX_RETRY_AFTER_S = 5.0 +# Hosted Jev reads images only behind this header (TypeSafe "danger zone", experimental). See #97. +JEV_IMAGES_HEADER = "TypeSafe-Danger-Zone-Images" +_TRUE = frozenset({"1", "true", "yes", "on"}) +_FALSE = frozenset({"", "0", "false", "no", "off"}) def decisions_timeout_from_env() -> float: @@ -59,6 +63,32 @@ def decisions_backend_from_env() -> str: return "typesafe" if os.getenv("TYPESAFE_API_KEY") and not os.getenv("TYPESAFE_API_URL") else "openrouter" +def jev_images_from_env(backend: str) -> bool: + """Whether ``S1A_JEV_IMAGES`` turns on screenshot input for hosted Jev; off by default. + + Only the direct TypeSafe backend is known to forward the ``assets`` map and the danger-zone header, so the + setting on the OpenRouter proxy (or a custom ``TYPESAFE_API_URL``) is a config error rather than a silent + text-only run. A value that is neither true nor false is a config error too. + """ + raw = (os.getenv("S1A_JEV_IMAGES") or "").strip().lower() + if raw in _FALSE: + return False + if raw not in _TRUE: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"S1A_JEV_IMAGES must be 1/true or 0/false, not {os.getenv('S1A_JEV_IMAGES')!r}", + ) + if backend != "typesafe": + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=( + "S1A_JEV_IMAGES needs the direct TypeSafe backend (TYPESAFE_API_KEY without TYPESAFE_API_URL); " + "the OpenRouter proxy is not known to forward image assets" + ), + ) + return True + + def client_from_env(backend: str, *, timeout_s: float) -> "JevDecisionsClient": """Build the decisions client for one backend from the environment. @@ -66,7 +96,9 @@ def client_from_env(backend: str, *, timeout_s: float) -> "JevDecisionsClient": overrides the pinned ``TYPESAFE_DEFAULT_MODEL``; an OpenRouter-style ``typesafe/...`` id there is a config error rather than a request TypeSafe would reject); ``openrouter`` talks to the proxy with ``OPENROUTER_API_KEY`` (``TYPESAFE_API_URL`` / ``TYPESAFE_MODEL`` override the proxy URL and model). + ``S1A_JEV_IMAGES`` (direct TypeSafe only) marks the client as able to carry screenshots. """ + images = jev_images_from_env(backend) if backend in DECISIONS_BACKENDS else False match backend: case "typesafe": model = (os.getenv("TYPESAFE_MODEL") or "").strip() or TYPESAFE_DEFAULT_MODEL @@ -84,6 +116,7 @@ def client_from_env(backend: str, *, timeout_s: float) -> "JevDecisionsClient": url=TYPESAFE_DECISIONS_URL, model=model, timeout_s=timeout_s, + images=images, ) case "openrouter": return JevDecisionsClient( @@ -107,7 +140,7 @@ def _retry_after_s(response: httpx.Response, *, fallback: float) -> float: class JevDecisionsClient: """Async HTTP client for the decisions endpoint with a warm keep-alive connection.""" - def __init__(self, *, api_key: str, url: str, model: str, timeout_s: float) -> None: + def __init__(self, *, api_key: str, url: str, model: str, timeout_s: float, images: bool = False) -> None: if not api_key: raise build_error( StatusCode.MODEL_SERVICE_CONFIG_ERROR, @@ -121,6 +154,7 @@ def __init__(self, *, api_key: str, url: str, model: str, timeout_s: float) -> N ) self.url = url self.model = model + self.images = images # JevModel sends screenshots (assets + the danger-zone header) only when True self._timeout_s = timeout_s # the deadline for one decision, retries included self._client = httpx.AsyncClient(timeout=timeout_s, headers={"Authorization": f"Bearer {api_key}"}) @@ -131,15 +165,19 @@ async def warm(self) -> None: except httpx.HTTPError: return None - async def decide(self, body: dict[str, Any]) -> tuple[dict[str, Any], int]: - """One decision: the payload and the last attempt's latency in ms; every retry fits one overall deadline.""" + async def decide( + self, body: dict[str, Any], *, headers: dict[str, str] | None = None + ) -> tuple[dict[str, Any], int]: + """One decision: the payload and the last attempt's latency in ms; every retry fits one overall deadline. + + ``headers`` are added to this request only (the image header rides only on bodies that carry an image).""" deadline = time.monotonic() + self._timeout_s attempt = 0 while True: started = time.perf_counter() remaining = deadline - time.monotonic() try: - response = await self._client.post(self.url, json=body, timeout=max(remaining, 0.01)) + response = await self._client.post(self.url, json=body, headers=headers, timeout=max(remaining, 0.01)) except _RETRIED_TRANSPORT as exc: attempt += 1 if attempt <= _TRANSPORT_RETRIES and time.monotonic() < deadline: diff --git a/tests/test_decision_models_jev_images.py b/tests/test_decision_models_jev_images.py new file mode 100644 index 0000000..39ab120 --- /dev/null +++ b/tests/test_decision_models_jev_images.py @@ -0,0 +1,206 @@ +# coding: utf-8 +"""Hosted Jev screenshots (#97, experimental): off by default, direct TypeSafe only, images prepared to the +documented limits before anything is sent.""" + +from __future__ import annotations + +import base64 +import io +import os +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, patch + +import httpx +from PIL import Image as PILImage + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import BaseError + +from s1a.agents import desktop +from s1a.decision_models import ChoiceQuestion, Image, JevModel, NoulQuestion, Observation, ScriptedTransport +from s1a.decision_models import wire +from s1a.decision_models.jev import JEV_IMAGE_MAX_SIDE, jev_image_asset +from s1a.tool import series + +HEADER = "TypeSafe-Danger-Zone-Images" +QUESTIONS = { + "pick": ChoiceQuestion({"a": "the left tile", "b": "the right tile"}, goal="click the red tile"), + "done": NoulQuestion("Is the task done?"), +} +_CLEAN_ENV = {"TYPESAFE_API_KEY": "", "TYPESAFE_API_URL": "", "OPENROUTER_API_KEY": "", "S1A_JEV_IMAGES": ""} + + +def _picture(size: tuple[int, int], fmt: str = "PNG") -> bytes: + buffer = io.BytesIO() + PILImage.new("RGB", size, (200, 30, 30)).save(buffer, format=fmt) + return buffer.getvalue() + + +def _image_transport() -> ScriptedTransport: + transport = ScriptedTransport() + transport.images = True + return transport + + +class TestSetting(TestCase): + def test_unset_is_off_and_text_only(self) -> None: + with patch.dict(os.environ, {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k"}): + client = wire.client_from_env("typesafe", timeout_s=5.0) + self.assertFalse(client.images) + self.assertFalse(JevModel(client).supports_images) + + def test_on_for_direct_typesafe(self) -> None: + for raw in ("1", "true", "YES", " on "): + with ( + self.subTest(raw=raw), + patch.dict(os.environ, {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k", "S1A_JEV_IMAGES": raw}), + ): + model = JevModel.from_env() + self.assertTrue(model.supports_images) + + def test_proxy_refuses_the_setting(self) -> None: + env = {**_CLEAN_ENV, "OPENROUTER_API_KEY": "k", "S1A_JEV_IMAGES": "1"} + with patch.dict(os.environ, env), self.assertRaises(BaseError) as caught: + JevModel.from_env() + self.assertEqual(caught.exception.code, StatusCode.MODEL_SERVICE_CONFIG_ERROR.code) + self.assertIn("TypeSafe", str(caught.exception)) + # a custom proxy URL next to a TypeSafe key is the proxy backend too + env = {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k", "TYPESAFE_API_URL": "http://proxy.test/", "S1A_JEV_IMAGES": "1"} + with patch.dict(os.environ, env), self.assertRaises(BaseError): + JevModel.from_env() + + def test_unreadable_value_is_a_config_error(self) -> None: + with patch.dict(os.environ, {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k", "S1A_JEV_IMAGES": "maybe"}): + with self.assertRaises(BaseError): + JevModel.from_env() + + +class TestRequests(IsolatedAsyncioTestCase): + async def test_text_only_request_is_unchanged_when_off(self) -> None: + transport = ScriptedTransport() + model = JevModel(transport) + observation = Observation({"title": "Canvas"}, images=(Image(_picture((400, 300))),)) + decision = await model.decide_many(observation, QUESTIONS) + self.assertEqual(set(transport.bodies[0]), {"model", "state", "questions"}) + self.assertEqual(transport.bodies[0]["state"], {"title": "Canvas"}) + self.assertEqual(transport.headers, [{}]) + self.assertNotIn("sent_image", decision.provenance) + + async def test_screenshot_rides_in_assets_with_the_header(self) -> None: + transport = _image_transport() + model = JevModel(transport) + png = _picture((640, 400)) + decision = await model.decide_many(Observation({"title": "Canvas"}, images=(Image(png),)), QUESTIONS) + body = transport.bodies[0] + self.assertEqual(transport.headers, [{HEADER: "true"}]) + (reference,) = body["assets"] + self.assertEqual(body["state"], {"title": "Canvas", "screenshot": reference}) + self.assertEqual( + body["assets"][reference], {"type": "image", "format": "png", "data_base64": base64.b64encode(png).decode()} + ) + # referenced exactly once (in the state), never per question + self.assertNotIn(reference, str(body["questions"])) + self.assertEqual( + decision.provenance["sent_image"], + {"width": 640, "height": 400, "bytes": len(png), "format": "png", "resized": False}, + ) + + async def test_fresh_reference_per_request(self) -> None: + transport = _image_transport() + model = JevModel(transport) + observation = Observation({}, images=(Image(_picture((300, 300))),)) + await model.decide_many(observation, QUESTIONS) + await model.decide_many(observation, QUESTIONS) + self.assertNotEqual(list(transport.bodies[0]["assets"]), list(transport.bodies[1]["assets"])) + + async def test_text_state_is_wrapped(self) -> None: + transport = _image_transport() + await JevModel(transport).decide_many(Observation("a page", images=(Image(_picture((300, 300))),)), QUESTIONS) + state = transport.bodies[0]["state"] + self.assertEqual(state["text"], "a page") + self.assertIn(state["screenshot"], transport.bodies[0]["assets"]) + + async def test_no_screenshot_means_a_plain_request(self) -> None: + transport = _image_transport() + await JevModel(transport).decide_many(Observation({"title": "x"}), QUESTIONS) + self.assertNotIn("assets", transport.bodies[0]) + self.assertEqual(transport.headers, [{}]) + + async def test_more_than_one_screenshot_fails_before_sending(self) -> None: + transport = _image_transport() + image = Image(_picture((300, 300))) + with self.assertRaises(BaseError) as caught: + await JevModel(transport).decide_many(Observation({}, images=(image, image)), QUESTIONS) + self.assertEqual(caught.exception.code, StatusCode.MODEL_CALL_FAILED.code) + self.assertEqual(transport.bodies, []) + + async def test_oversize_is_resized_and_undersize_rejected(self) -> None: + transport = _image_transport() + decision = await JevModel(transport).decide_many( + Observation({}, images=(Image(_picture((3136, 1960))),)), QUESTIONS + ) + sent = decision.provenance["sent_image"] + self.assertEqual((sent["width"], sent["height"], sent["resized"]), (JEV_IMAGE_MAX_SIDE, 980, True)) + (asset,) = transport.bodies[0]["assets"].values() + with PILImage.open(io.BytesIO(base64.b64decode(asset["data_base64"]))) as picture: + self.assertEqual(picture.size, (JEV_IMAGE_MAX_SIDE, 980)) + for size in ((279, 600), (4000, 500)): # too small, or too thin once the long side fits + with self.subTest(size=size), self.assertRaises(BaseError) as caught: + await JevModel(_image_transport()).decide_many( + Observation({}, images=(Image(_picture(size)),)), QUESTIONS + ) + self.assertEqual(caught.exception.code, StatusCode.MODEL_SERVICE_CONFIG_ERROR.code) + + async def test_bad_images_are_rejected(self) -> None: + cases = ( + Image(_picture((300, 300), "GIF"), "image/gif"), + Image(_picture((300, 300), "JPEG"), "image/png"), # bytes do not match the media type + Image(b"not an image"), + ) + for image in cases: + with self.subTest(media_type=image.media_type), self.assertRaises(BaseError): + jev_image_asset(image) + asset, sent = jev_image_asset(Image(_picture((300, 300), "JPEG"), "image/jpeg")) + self.assertEqual((asset["format"], sent["format"]), ("jpg", "jpg")) + + +class TestWireHeader(IsolatedAsyncioTestCase): + async def test_header_only_on_the_request_that_asks_for_it(self) -> None: + seen: list[httpx.Headers] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers) + return httpx.Response(200, json={"answers": {}}) + + client = wire.JevDecisionsClient(api_key="k", url="https://api.test/v1/systemone", model="m", timeout_s=5.0) + client._client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), headers={"Authorization": "Bearer k"} + ) + await client.decide({"model": "m"}, headers={HEADER: "true"}) + await client.decide({"model": "m"}) + await client.close() + self.assertEqual(seen[0][HEADER], "true") + self.assertNotIn(HEADER, seen[1]) + self.assertEqual(seen[0]["Authorization"], "Bearer k") + + +class TestDesktopPixelTargets(TestCase): + def _args(self): + return series.parser(desktop.SPEC).parse_args( + [ + "--model", "jev", "--rethink", "off", "--episodes", "1", "--app", "Canvas", + "--goal", "click Save", "--expect", "Saved", "--pixel-target", "left=0.25,0.5", + ] + ) # fmt: skip + + def test_jev_with_images_may_use_pixel_targets(self) -> None: + env = {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k", "S1A_JEV_IMAGES": "1"} + with patch.dict(os.environ, env), patch.object(desktop, "driver_from_env", return_value=AsyncMock()): + desktop.make_series(self._args()) + + def test_jev_without_images_still_refuses(self) -> None: + with ( + patch.dict(os.environ, {**_CLEAN_ENV, "TYPESAFE_API_KEY": "k"}), + self.assertRaisesRegex(ValueError, "S1A_JEV_IMAGES"), + ): + desktop.make_series(self._args())