From 2e05762018ed75502f259ead3c9fd5d4136a2098 Mon Sep 17 00:00:00 2001 From: SiddarthAA Date: Fri, 28 Aug 2026 13:00:49 +0530 Subject: [PATCH 01/20] docs(sdk): clarify evaluator v2 boundary --- sdk/python/CHANGELOG.md | 3 +++ sdk/python/README.md | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 57aa24a9..5e763997 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,6 +16,9 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. +- Document that the retired inbound evaluator package is not part of this SDK + and reserve evaluator authoring for the forthcoming outbound-only v2 runtime. + ## 0.0.1b1 — 2026-08-24 The first release under this name. Everything below describes the package as it diff --git a/sdk/python/README.md b/sdk/python/README.md index f974587d..e846b623 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -10,6 +10,15 @@ the platform. - **Dependencies:** none. Standard library only, so installing it constrains nothing else in your environment. +## Evaluator v2 status + +This package currently provides tracing and event emission only. The legacy +inbound `agenteye-evaluator` package has been retired; do not build new evaluator +services against its server-push HTTP contract. A customer-hosted, outbound-only +worker runtime will be added under the lazy `failproofai_sdk.evaluator` namespace +as part of Evaluator v2. Until that API ships, no evaluator module is included in +the distribution. + ## Installation ```bash From fa8a3d21dcbfb4b724691e53a8df0a81f302e6ab Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 16:04:04 +0530 Subject: [PATCH 02/20] feat(sdk): add evaluator v2 worker runtime --- sdk/python/CHANGELOG.md | 5 +- sdk/python/README.md | 11 +- sdk/python/examples/evaluator_worker.py | 111 +++ .../failproofai_sdk/evaluator/__init__.py | 85 +++ .../failproofai_sdk/evaluator/__main__.py | 49 ++ .../failproofai_sdk/evaluator/authoring.py | 377 ++++++++++ .../failproofai_sdk/evaluator/client.py | 268 +++++++ .../failproofai_sdk/evaluator/protocol.py | 659 ++++++++++++++++ .../failproofai_sdk/evaluator/runtime.py | 563 ++++++++++++++ .../tests/fixtures/evaluator_v2/README.md | 28 + .../tests/fixtures/evaluator_v2/contract.json | 225 ++++++ sdk/python/tests/test_evaluator_authoring.py | 124 ++++ sdk/python/tests/test_evaluator_client.py | 221 ++++++ sdk/python/tests/test_evaluator_example.py | 35 + sdk/python/tests/test_evaluator_http_e2e.py | 631 ++++++++++++++++ sdk/python/tests/test_evaluator_main.py | 47 ++ sdk/python/tests/test_evaluator_protocol.py | 223 ++++++ sdk/python/tests/test_evaluator_runtime.py | 701 ++++++++++++++++++ sdk/python/tests/test_zero_dependencies.py | 17 + 19 files changed, 4372 insertions(+), 8 deletions(-) create mode 100644 sdk/python/examples/evaluator_worker.py create mode 100644 sdk/python/failproofai_sdk/evaluator/__init__.py create mode 100644 sdk/python/failproofai_sdk/evaluator/__main__.py create mode 100644 sdk/python/failproofai_sdk/evaluator/authoring.py create mode 100644 sdk/python/failproofai_sdk/evaluator/client.py create mode 100644 sdk/python/failproofai_sdk/evaluator/protocol.py create mode 100644 sdk/python/failproofai_sdk/evaluator/runtime.py create mode 100644 sdk/python/tests/fixtures/evaluator_v2/README.md create mode 100644 sdk/python/tests/fixtures/evaluator_v2/contract.json create mode 100644 sdk/python/tests/test_evaluator_authoring.py create mode 100644 sdk/python/tests/test_evaluator_client.py create mode 100644 sdk/python/tests/test_evaluator_example.py create mode 100644 sdk/python/tests/test_evaluator_http_e2e.py create mode 100644 sdk/python/tests/test_evaluator_main.py create mode 100644 sdk/python/tests/test_evaluator_protocol.py create mode 100644 sdk/python/tests/test_evaluator_runtime.py diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 5e763997..74d6b5b3 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,8 +16,9 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. -- Document that the retired inbound evaluator package is not part of this SDK - and reserve evaluator authoring for the forthcoming outbound-only v2 runtime. +- Retire the old inbound evaluator boundary and add evaluator authoring plus the + outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator` + namespace. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/README.md b/sdk/python/README.md index e846b623..3cb8adaf 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -12,12 +12,11 @@ the platform. ## Evaluator v2 status -This package currently provides tracing and event emission only. The legacy -inbound `agenteye-evaluator` package has been retired; do not build new evaluator -services against its server-push HTTP contract. A customer-hosted, outbound-only -worker runtime will be added under the lazy `failproofai_sdk.evaluator` namespace -as part of Evaluator v2. Until that API ships, no evaluator module is included in -the distribution. +The legacy inbound `agenteye-evaluator` package has been retired; do not build new +evaluator services against its server-push HTTP contract. Evaluator v2 authoring +and its customer-hosted, outbound-only worker runtime live under the lazy +`failproofai_sdk.evaluator` namespace. Importing the top-level tracing SDK does not +import or start the evaluator runtime. ## Installation diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py new file mode 100644 index 00000000..a0cea74b --- /dev/null +++ b/sdk/python/examples/evaluator_worker.py @@ -0,0 +1,111 @@ +"""Customer evaluator with deterministic and optional async judge checks.""" + +from __future__ import annotations + +import asyncio +import json +import os +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator import ( + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, +) + +app = Evaluator(name="customer-production", version="2026.08.1") + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +@app.eval( + "tool_efficiency", + version="1.0.0", + labels=["tools", "deterministic"], + when=lambda session: ConditionResult( + session.count("tool_use") > 0, "no_tool_calls" + ), +) +def tool_efficiency(session): + calls = session.events_of_type("tool_use") + distinct = { + event.payload.get("tool_name") + for event in calls + if event.payload.get("tool_name") + } + value = len(distinct) / len(calls) + return EvalResult( + score=Score(value, passed=value >= 0.7), + metrics={ + "tool_call_count": Metric(len(calls), unit="events"), + "distinct_tool_count": Metric(len(distinct), unit="tools"), + }, + reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls", + ) + + +def _judge_configured(session): + configured = bool(os.environ.get("EXAMPLE_JUDGE_URL")) + return ConditionResult(configured, "judge_not_configured") + + +def _last_content(session, event_type): + events = session.events_of_type(event_type) + if not events: + return None + payload = events[-1].payload + fields = { + "human_input": ("response",), + "model_response": ("content",), + "agent_end": ("summary",), + }.get(event_type, ("content", "summary", "response")) + return next((payload.get(field) for field in fields if payload.get(field)), None) + + +def _call_judge(question, answer): + url = os.environ["EXAMPLE_JUDGE_URL"] + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + token = os.environ.get("EXAMPLE_JUDGE_TOKEN") + body = json.dumps({"question": question, "answer": answer}).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + request = Request(url, data=body, headers=headers, method="POST") + with build_opener(_RejectRedirects()).open(request, timeout=25) as response: # nosec B310 + result = json.loads(response.read(64 * 1024)) + return float(result["score"]), str( + result.get("reasoning") or "Judge returned no reasoning" + ) + + +@app.eval( + "answer_relevance", + version="judge-api-v1", + labels=["llm_judge", "relevance"], + when=_judge_configured, + timeout_seconds=30, +) +async def answer_relevance(session): + question = _last_content(session, "human_input") + answer = _last_content(session, "model_response") + if question is None or answer is None: + raise ValueError("answer relevance requires human input and model output") + value, reasoning = await asyncio.to_thread(_call_judge, question, answer) + value = min(max(value, 0.0), 1.0) + return EvalResult( + score=Score(value, passed=value >= 0.7), + reasoning=reasoning, + labels=("llm_judge", "relevance"), + ) + + +if __name__ == "__main__": + app.run_from_env() diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py new file mode 100644 index 00000000..dff0cad6 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -0,0 +1,85 @@ +"""Authoring and worker primitives for FailproofAI Evaluator v2. + +This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users +who only emit telemetry do not import evaluator networking or runtime code. +""" + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, + Metric, + Score, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + Assignment, + CatalogDefinition, + ClaimRequest, + ClaimResponse, + ErrorResponse, + EvalSelection, + EvaluatorKind, + HeartbeatRequest, + HeartbeatResponse, + HeartbeatRun, + PlannedRun, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + RemoteError, + ResultItem, + ResultKind, + ResultRequest, + ResultResponse, + SessionTranscript, + SkippedEval, + TerminalRunStatus, + TranscriptEvent, + UnsupportedProtocolVersion, +) +from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + +__all__ = [ + "Assertion", + "Assignment", + "CatalogDefinition", + "ClaimRequest", + "ClaimResponse", + "ConditionResult", + "ErrorResponse", + "EvalDefinition", + "EvalResult", + "EvalSelection", + "Evaluator", + "EvaluatorAPIError", + "EvaluatorClient", + "EvaluatorKind", + "HeartbeatRequest", + "HeartbeatResponse", + "HeartbeatRun", + "Metric", + "PlanRequest", + "PlanResponse", + "PlannedRun", + "ProtocolError", + "RegisterRequest", + "RegisterResponse", + "RemoteError", + "ResultItem", + "ResultKind", + "ResultRequest", + "ResultResponse", + "Score", + "SessionTranscript", + "SkippedEval", + "TerminalRunStatus", + "TranscriptEvent", + "UnsupportedProtocolVersion", + "WorkerConfig", + "WorkerRuntime", +] diff --git a/sdk/python/failproofai_sdk/evaluator/__main__.py b/sdk/python/failproofai_sdk/evaluator/__main__.py new file mode 100644 index 00000000..f7bff1c5 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__main__.py @@ -0,0 +1,49 @@ +"""Run an evaluator declared as ``module:attribute``.""" + +from __future__ import annotations + +import argparse +import importlib +import os +from collections.abc import Sequence + +from failproofai_sdk.evaluator.authoring import Evaluator + + +def load_evaluator(spec: str) -> Evaluator: + module_name, separator, attribute = spec.partition(":") + if not module_name: + raise ValueError("evaluator module must not be empty") + if not separator: + attribute = "app" + if not attribute: + raise ValueError("evaluator attribute must not be empty") + module = importlib.import_module(module_name) + try: + evaluator = getattr(module, attribute) + except AttributeError as error: + raise ValueError(f"{spec!r} does not define {attribute!r}") from error + if not isinstance(evaluator, Evaluator): + raise TypeError( + f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator" + ) + return evaluator + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator") + parser.add_argument( + "module", + nargs="?", + default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"), + help="Python module and optional attribute (for example my_evals:app)", + ) + args = parser.parse_args(argv) + if not args.module: + parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)") + load_evaluator(args.module).run_from_env() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/failproofai_sdk/evaluator/authoring.py b/sdk/python/failproofai_sdk/evaluator/authoring.py new file mode 100644 index 00000000..db1788d9 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/authoring.py @@ -0,0 +1,377 @@ +"""Evaluator definition registry and typed author results.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import re +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from failproofai_sdk.evaluator.protocol import ( + MAX_CATALOG_DEFINITIONS, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABEL_BYTES, + MAX_LABELS_PER_RESULT, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + CatalogDefinition, + ResultItem, + ResultKind, + SessionTranscript, +) + +_KEY = re.compile(r"^[a-z][a-z0-9_]*$") +EvalFunction = Callable[[SessionTranscript], "EvalResult | Awaitable[EvalResult]"] +ConditionFunction = Callable[ + [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]" +] +CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"] + + +def _bounded(value: str, *, field_name: str, maximum: int) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + if not value: + raise ValueError(f"{field_name} must not be empty") + size = len(value.encode("utf-8")) + if size > maximum: + raise ValueError(f"{field_name} is {size} bytes; maximum is {maximum}") + return value + + +def _finite(value: float, field_name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{field_name} must be a number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field_name} must be finite") + return result + + +def _labels(values: tuple[str, ...] | list[str]) -> tuple[str, ...]: + if len(values) > MAX_LABELS_PER_RESULT: + raise ValueError(f"at most {MAX_LABELS_PER_RESULT} labels are allowed") + normalized = [] + for label in values: + normalized.append(_bounded(label, field_name="label", maximum=MAX_LABEL_BYTES)) + if len(set(normalized)) != len(normalized): + raise ValueError("labels must be unique") + return tuple(sorted(normalized)) + + +@dataclass(frozen=True) +class Score: + value: float + passed: bool | None = None + unit: str = "ratio" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + value = _finite(self.value, "score value") + if not 0 <= value <= 1: + raise ValueError("score value must be between 0 and 1") + object.__setattr__(self, "value", value) + if self.passed is not None and not isinstance(self.passed, bool): + raise TypeError("score passed must be a boolean or None") + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Metric: + value: float + unit: str = "" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "value", _finite(self.value, "metric value")) + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Assertion: + passed: bool + description: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.passed, bool): + raise TypeError("assertion passed must be a boolean") + if self.description is not None: + _bounded( + self.description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class ConditionResult: + applicable: bool + reason_code: str = "condition_false" + + def __post_init__(self) -> None: + if not isinstance(self.applicable, bool): + raise TypeError("condition applicable must be a boolean") + _validate_key(self.reason_code, "condition reason code") + + +def _validate_result_text( + unit: str, display_value: str | None, description: str | None +) -> None: + if unit: + _bounded(unit, field_name="unit", maximum=MAX_UNIT_BYTES) + if display_value is not None: + _bounded( + display_value, + field_name="display value", + maximum=MAX_DISPLAY_VALUE_BYTES, + ) + if description is not None: + _bounded( + description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class EvalResult: + score: Score | None = None + metrics: Mapping[str, Metric | float] = field(default_factory=dict) + assertions: Mapping[str, Assertion | bool] = field(default_factory=dict) + reasoning: str | None = None + summary: str | None = None + labels: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.reasoning is not None: + _bounded( + self.reasoning, + field_name="reasoning", + maximum=MAX_REASONING_BYTES, + ) + if self.summary is not None: + _bounded(self.summary, field_name="summary", maximum=MAX_SUMMARY_BYTES) + object.__setattr__(self, "labels", _labels(list(self.labels))) + + def result_items(self, eval_key: str) -> tuple[ResultItem, ...]: + items: list[ResultItem] = [] + if self.score is not None: + items.append( + ResultItem( + result_key=eval_key, + result_kind=ResultKind.SCORE, + numeric_value=self.score.value, + bool_value=self.score.passed, + unit=self.score.unit, + display_value=self.score.display_value, + description=self.score.description, + reasoning=self.reasoning, + labels=self.labels, + ) + ) + for key, raw_metric in sorted(self.metrics.items()): + _validate_key(key, "metric key") + metric = ( + raw_metric if isinstance(raw_metric, Metric) else Metric(raw_metric) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.METRIC, + numeric_value=metric.value, + unit=metric.unit, + display_value=metric.display_value, + description=metric.description, + labels=self.labels, + ) + ) + for key, raw_assertion in sorted(self.assertions.items()): + _validate_key(key, "assertion key") + assertion = ( + raw_assertion + if isinstance(raw_assertion, Assertion) + else Assertion(raw_assertion) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.ASSERTION, + bool_value=assertion.passed, + description=assertion.description, + labels=self.labels, + ) + ) + if not items: + raise ValueError("an EvalResult must contain a score, metric, or assertion") + if len(items) > MAX_RESULTS_PER_RUN: + raise ValueError( + f"an EvalResult may contain at most {MAX_RESULTS_PER_RUN} results" + ) + keys = [item.result_key for item in items] + if len(keys) != len(set(keys)): + raise ValueError("result keys must be unique within one evaluation run") + return tuple(items) + + +def _validate_key(value: str, field_name: str = "eval_key") -> str: + _bounded(value, field_name=field_name, maximum=MAX_EVAL_KEY_BYTES) + if not _KEY.fullmatch(value): + raise ValueError(f"{field_name} must match {_KEY.pattern}") + return value + + +@dataclass(frozen=True) +class EvalDefinition: + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] + function: EvalFunction + condition: ConditionFunction | None + on_cancel: CancellationFunction | None + timeout_seconds: float | None + + def catalog_definition(self) -> CatalogDefinition: + return CatalogDefinition( + eval_key=self.eval_key, + display_name=self.display_name, + eval_version=self.eval_version, + result_kind=self.result_kind, + labels=self.labels, + ) + + +class Evaluator: + """A process-local collection of explicitly versioned evaluations.""" + + def __init__(self, *, name: str, version: str) -> None: + self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES) + self.version = _bounded( + version, field_name="version", maximum=MAX_VERSION_BYTES + ) + self._definitions: dict[str, EvalDefinition] = {} + + def eval( + self, + eval_key: str, + *, + version: str, + display_name: str | None = None, + result_kind: ResultKind | str = ResultKind.SCORE, + labels: tuple[str, ...] | list[str] = (), + when: ConditionFunction | None = None, + on_cancel: CancellationFunction | None = None, + timeout_seconds: float | None = None, + ) -> Callable[[EvalFunction], EvalFunction]: + key = _validate_key(eval_key) + eval_version = _bounded( + version, field_name="eval version", maximum=MAX_VERSION_BYTES + ) + display = _bounded( + display_name or eval_key.replace("_", " ").capitalize(), + field_name="display name", + maximum=MAX_DISPLAY_NAME_BYTES, + ) + kind = ResultKind(result_kind) + normalized_labels = _labels(list(labels)) + if timeout_seconds is not None: + timeout_seconds = _finite(timeout_seconds, "timeout_seconds") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + + def register(function: EvalFunction) -> EvalFunction: + if key in self._definitions: + raise ValueError(f"duplicate eval key: {key}") + if len(self._definitions) >= MAX_CATALOG_DEFINITIONS: + raise ValueError( + f"an evaluator may define at most {MAX_CATALOG_DEFINITIONS} evaluations" + ) + if not callable(function): + raise TypeError("evaluation must be callable") + if when is not None and not callable(when): + raise TypeError("when must be callable") + if on_cancel is not None and not callable(on_cancel): + raise TypeError("on_cancel must be callable") + self._definitions[key] = EvalDefinition( + eval_key=key, + display_name=display, + eval_version=eval_version, + result_kind=kind, + labels=normalized_labels, + function=function, + condition=when, + on_cancel=on_cancel, + timeout_seconds=timeout_seconds, + ) + return function + + return register + + @property + def definitions(self) -> tuple[EvalDefinition, ...]: + return tuple(self._definitions[key] for key in sorted(self._definitions)) + + def catalog(self) -> tuple[CatalogDefinition, ...]: + return tuple(definition.catalog_definition() for definition in self.definitions) + + @property + def catalog_revision(self) -> str: + payload = [item.to_wire() for item in self.catalog()] + canonical = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(canonical).hexdigest() + + def definition(self, eval_key: str) -> EvalDefinition: + try: + return self._definitions[eval_key] + except KeyError as error: + raise KeyError(f"unknown eval key: {eval_key}") from error + + def run_from_env(self) -> None: + """Run this evaluator until the process receives a stop request.""" + import asyncio + import signal + + from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + + async def run() -> None: + runtime = WorkerRuntime(self, WorkerConfig.from_env()) + loop = asyncio.get_running_loop() + for name in ("SIGINT", "SIGTERM"): + process_signal = getattr(signal, name, None) + if process_signal is None: + continue + try: + loop.add_signal_handler(process_signal, runtime.stop) + except (NotImplementedError, RuntimeError): + pass + await runtime.run_forever() + + asyncio.run(run()) + + @staticmethod + async def call( + function: EvalFunction | ConditionFunction, session: SessionTranscript + ) -> Any: + result = function(session) + if inspect.isawaitable(result): + return await result + return result diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py new file mode 100644 index 00000000..60fa32c2 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -0,0 +1,268 @@ +"""Standard-library HTTP client for the Evaluator v2 worker protocol.""" + +from __future__ import annotations + +import json +import random +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator.protocol import ( + CLAIM_PATH, + HEARTBEAT_PATH, + LEASE_GENERATION_HEADER, + MAX_TRANSCRIPT_BYTES, + PLAN_PATH, + REGISTER_PATH, + RESULT_PATH, + WORKER_ID_HEADER, + Assignment, + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + WireModel, +) + +_DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024 +_RETRYABLE_HTTP_STATUSES = frozenset({429, 502, 503, 504}) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def _open_without_redirects(request: Request, *, timeout: float): + return build_opener(_RejectRedirects()).open(request, timeout=timeout) + + +class EvaluatorAPIError(RuntimeError): + def __init__( + self, + *, + status: int | None, + code: str, + message: str, + retryable: bool, + request_id: str | None = None, + ) -> None: + super().__init__(f"{code}: {message}") + self.status = status + self.code = code + self.retryable = retryable + self.request_id = request_id + + +class EvaluatorClient: + """Direct server client; evaluator traffic never passes through the dashboard.""" + + def __init__( + self, + *, + base_url: str, + credential: str, + timeout_seconds: float = 30, + max_retries: int = 3, + opener: Callable[..., Any] | None = None, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + parsed = urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("base_url must be an absolute http(s) URL") + if not credential or not credential.strip(): + raise ValueError("credential must not be empty") + if any( + ord(character) < 32 or ord(character) == 127 for character in credential + ): + raise ValueError("credential must not contain control characters") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if max_retries < 0: + raise ValueError("max_retries must not be negative") + self._base_url = base_url.rstrip("/") + "/" + self._origin = (parsed.scheme, parsed.netloc) + self._credential = credential + self._timeout_seconds = timeout_seconds + self._max_retries = max_retries + self._opener = opener or _open_without_redirects + self._sleeper = sleeper + + def register(self, request: RegisterRequest) -> RegisterResponse: + return RegisterResponse.from_wire( + self._json("POST", REGISTER_PATH, request, retry=True) + ) + + def claim(self, request: ClaimRequest) -> ClaimResponse: + # A lost claim response may already have leased work. Do not hide a + # second claim behind transport retry; the runtime recalculates capacity. + return ClaimResponse.from_wire( + self._json("POST", CLAIM_PATH, request, retry=False) + ) + + def transcript( + self, assignment: Assignment, *, worker_id: str + ) -> SessionTranscript: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + return SessionTranscript.from_wire( + self._json( + "GET", + assignment.transcript_url, + None, + retry=True, + headers=headers, + response_limit=MAX_TRANSCRIPT_BYTES, + ) + ) + + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: + return PlanResponse.from_wire( + self._json( + "POST", + PLAN_PATH.format(assignment_id=assignment_id), + request, + retry=True, + ) + ) + + def heartbeat(self, request: HeartbeatRequest) -> HeartbeatResponse: + return HeartbeatResponse.from_wire( + self._json("POST", HEARTBEAT_PATH, request, retry=True) + ) + + def submit_result(self, run_id: str, request: ResultRequest) -> ResultResponse: + return ResultResponse.from_wire( + self._json( + "POST", + RESULT_PATH.format(evaluation_run_id=run_id), + request, + retry=True, + ) + ) + + def _url(self, path: str) -> str: + url = urljoin(self._base_url, path) + parsed = urlsplit(url) + if (parsed.scheme, parsed.netloc) != self._origin: + raise EvaluatorAPIError( + status=None, + code="invalid_transcript_url", + message="server supplied a URL outside the configured API origin", + retryable=False, + ) + return url + + def _json( + self, + method: str, + path: str, + body: WireModel | None, + *, + retry: bool, + headers: Mapping[str, str] | None = None, + response_limit: int = _DEFAULT_RESPONSE_LIMIT, + ) -> dict[str, Any]: + encoded = None + request_headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self._credential}", + "User-Agent": "failproofai-sdk-evaluator/2", + } + if body is not None: + encoded = json.dumps( + body.to_wire(), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + + attempts = self._max_retries + 1 if retry else 1 + for attempt in range(attempts): + request = Request( + self._url(path), data=encoded, headers=request_headers, method=method + ) + try: + with self._opener(request, timeout=self._timeout_seconds) as response: + return self._decode( + response.read(response_limit + 1), response_limit + ) + except HTTPError as error: + api_error = self._http_error(error, response_limit) + if attempt + 1 == attempts or not api_error.retryable: + raise api_error from error + except (URLError, TimeoutError, OSError) as error: + if attempt + 1 == attempts: + raise EvaluatorAPIError( + status=None, + code="transport_error", + message=str(error), + retryable=True, + ) from error + # Jitter is scheduling noise, not a security decision. + self._sleeper(random.uniform(0, min(0.25 * (2**attempt), 2.0))) # nosec B311 + raise AssertionError("retry loop exhausted without returning or raising") + + @staticmethod + def _decode(raw: bytes, limit: int) -> dict[str, Any]: + if len(raw) > limit: + raise EvaluatorAPIError( + status=None, + code="response_too_large", + message=f"server response exceeds {limit} bytes", + retryable=False, + ) + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response was not valid JSON", + retryable=False, + ) from error + if not isinstance(value, dict): + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response must be a JSON object", + retryable=False, + ) + return value + + @classmethod + def _http_error(cls, error: HTTPError, limit: int) -> EvaluatorAPIError: + raw = error.read(limit + 1) + try: + response = ErrorResponse.from_wire(cls._decode(raw, limit)) + except (ValueError, EvaluatorAPIError): + return EvaluatorAPIError( + status=error.code, + code="http_error", + message=f"server returned HTTP {error.code}", + retryable=error.code in _RETRYABLE_HTTP_STATUSES, + ) + return EvaluatorAPIError( + status=error.code, + code=response.error.code, + message=response.error.message, + retryable=response.error.retryable, + request_id=response.error.request_id, + ) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py new file mode 100644 index 00000000..09adf8be --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -0,0 +1,659 @@ +"""Dependency-free wire models for the outbound Evaluator v2 protocol.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any, TypeVar + +PROTOCOL_VERSION = "2" +TRANSCRIPT_SCHEMA_VERSION = "2" +RESULT_SCHEMA_VERSION = "2" + +REGISTER_PATH = "/v1/evaluator/workers/register" +CLAIM_PATH = "/v1/evaluator/assignments/claim" +TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" +HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" +RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" +WORKER_ID_HEADER = "X-FailproofAI-Worker-Id" +LEASE_GENERATION_HEADER = "X-FailproofAI-Lease-Generation" + +HEARTBEAT_INTERVAL_SECONDS = 30 +LEASE_DURATION_SECONDS = 120 +MAX_CLAIM_WAIT_SECONDS = 25 +MAX_ATTEMPTS = 5 + +MAX_CATALOG_DEFINITIONS = 100 +MAX_CLAIM_CAPACITY = 32 +MAX_TRANSCRIPT_BYTES = 25 * 1024 * 1024 +MAX_RESULTS_PER_RUN = 25 +MAX_EVAL_KEY_BYTES = 128 +MAX_DISPLAY_NAME_BYTES = 128 +MAX_VERSION_BYTES = 128 +MAX_WORKER_ID_BYTES = 128 +MAX_LABEL_BYTES = 64 +MAX_LABELS_PER_RESULT = 20 +MAX_SUMMARY_BYTES = 4 * 1024 +MAX_REASONING_BYTES = 16 * 1024 +MAX_UNIT_BYTES = 64 +MAX_DISPLAY_VALUE_BYTES = 256 +MAX_DESCRIPTION_BYTES = 1_000 +MAX_ERROR_CODE_BYTES = 64 +MAX_ERROR_MESSAGE_BYTES = 4 * 1024 + +ERROR_SPECS = { + "invalid_credentials": {"http_status": 401, "retryable": False}, + "instance_disabled": {"http_status": 403, "retryable": False}, + "assignment_not_found": {"http_status": 404, "retryable": False}, + "run_not_found": {"http_status": 404, "retryable": False}, + "catalog_mismatch": {"http_status": 409, "retryable": False}, + "lease_lost": {"http_status": 409, "retryable": False}, + "plan_conflict": {"http_status": 409, "retryable": False}, + "submission_conflict": {"http_status": 409, "retryable": False}, + "retry_budget_exhausted": {"http_status": 409, "retryable": False}, + "transcript_too_large": {"http_status": 413, "retryable": False}, + "invalid_request": {"http_status": 422, "retryable": False}, + "invalid_catalog": {"http_status": 422, "retryable": False}, + "unsupported_protocol_version": {"http_status": 426, "retryable": False}, + "internal_error": {"http_status": 500, "retryable": True}, +} + + +class ProtocolError(ValueError): + """A local or remote evaluator protocol contract violation.""" + + +class UnsupportedProtocolVersion(ProtocolError): + def __init__(self, received: str) -> None: + super().__init__( + f"unsupported evaluator protocol version {received!r}; " + f"supported major version is {PROTOCOL_VERSION}" + ) + self.received = received + + +def validate_protocol_version(version: str) -> None: + if version != PROTOCOL_VERSION: + raise UnsupportedProtocolVersion(version) + + +class EvaluatorKind(str, Enum): + MANAGED = "managed" + CUSTOMER = "customer" + + +class ResultKind(str, Enum): + SCORE = "score" + METRIC = "metric" + ASSERTION = "assertion" + + +class TerminalRunStatus(str, Enum): + SUCCEEDED = "succeeded" + FAILED = "failed" + TIMED_OUT = "timed_out" + CANCELLED = "cancelled" + + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +def _wire(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if hasattr(value, "__dataclass_fields__"): + return {key: _wire(item) for key, item in asdict(value).items()} + if isinstance(value, (list, tuple)): + return [_wire(item) for item in value] + if isinstance(value, dict): + return {key: _wire(item) for key, item in value.items()} + return value + + +class WireModel: + def to_wire(self) -> dict[str, Any]: + return _wire(self) + + +def _string(data: Mapping[str, Any], key: str) -> str: + value = data.get(key) + if not isinstance(value, str): + raise ProtocolError(f"{key} must be a string") + return value + + +def _integer(data: Mapping[str, Any], key: str) -> int: + value = data.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{key} must be an integer") + return value + + +def _list(data: Mapping[str, Any], key: str) -> list[Any]: + value = data.get(key) + if not isinstance(value, list): + raise ProtocolError(f"{key} must be an array") + return value + + +def _object(value: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ProtocolError(f"{field_name} must be an object") + return value + + +def _object_list(data: Mapping[str, Any], key: str) -> tuple[Mapping[str, Any], ...]: + return tuple( + _object(value, f"{key}[{index}]") + for index, value in enumerate(_list(data, key)) + ) + + +def _string_list(data: Mapping[str, Any], key: str) -> tuple[str, ...]: + values = _list(data, key) + for index, value in enumerate(values): + if not isinstance(value, str): + raise ProtocolError(f"{key}[{index}] must be a string") + return tuple(values) + + +def _enum(enum_type: type[_EnumT], data: Mapping[str, Any], key: str) -> _EnumT: + value = _string(data, key) + try: + return enum_type(value) + except ValueError as error: + allowed = ", ".join(repr(item.value) for item in enum_type) + raise ProtocolError(f"{key} must be one of {allowed}") from error + + +def _positive_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value <= 0: + raise ProtocolError(f"{key} must be greater than zero") + return value + + +def _nonnegative_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value < 0: + raise ProtocolError(f"{key} must not be negative") + return value + + +def _optional_string(data: Mapping[str, Any], key: str) -> str | None: + value = data.get(key) + if value is not None and not isinstance(value, str): + raise ProtocolError(f"{key} must be a string or null") + return value + + +@dataclass(frozen=True) +class CatalogDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> CatalogDefinition: + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class RegisterRequest(WireModel): + worker_id: str + sdk_version: str + catalog_revision: str + max_concurrency: int + definitions: tuple[CatalogDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + sdk_version=_string(data, "sdk_version"), + catalog_revision=_string(data, "catalog_revision"), + max_concurrency=_integer(data, "max_concurrency"), + definitions=tuple( + CatalogDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class RegisterResponse(WireModel): + evaluator_instance_id: str + evaluator_kind: EvaluatorKind + heartbeat_interval_seconds: int + lease_duration_seconds: int + claim_limit: int + disabled_definitions: tuple[str, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + evaluator_instance_id=_string(data, "evaluator_instance_id"), + evaluator_kind=_enum(EvaluatorKind, data, "evaluator_kind"), + heartbeat_interval_seconds=_integer(data, "heartbeat_interval_seconds"), + lease_duration_seconds=_integer(data, "lease_duration_seconds"), + claim_limit=_integer(data, "claim_limit"), + disabled_definitions=_string_list(data, "disabled_definitions"), + ) + + +@dataclass(frozen=True) +class ClaimRequest(WireModel): + worker_id: str + catalog_revision: str + capacity: int + wait_seconds: int + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + catalog_revision=_string(data, "catalog_revision"), + capacity=_integer(data, "capacity"), + wait_seconds=_integer(data, "wait_seconds"), + ) + + +@dataclass(frozen=True) +class Assignment(WireModel): + assignment_id: str + lease_generation: int + lease_expires_at: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + trigger_reason: str + event_count: int + transcript_url: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> Assignment: + return cls( + assignment_id=_string(data, "assignment_id"), + lease_generation=_positive_integer(data, "lease_generation"), + lease_expires_at=_string(data, "lease_expires_at"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + trigger_reason=_string(data, "trigger_reason"), + event_count=_nonnegative_integer(data, "event_count"), + transcript_url=_string(data, "transcript_url"), + ) + + +@dataclass(frozen=True) +class ClaimResponse(WireModel): + assignments: tuple[Assignment, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + tuple( + Assignment.from_wire(item) for item in _object_list(data, "assignments") + ) + ) + + +@dataclass(frozen=True) +class TranscriptEvent(WireModel): + id: str + ts: str + event_type: str + payload: Mapping[str, Any] + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> TranscriptEvent: + payload = data.get("payload") + if not isinstance(payload, Mapping): + raise ProtocolError("payload must be an object") + return cls( + id=_string(data, "id"), + ts=_string(data, "ts"), + event_type=_string(data, "event_type"), + payload=dict(payload), + ) + + +@dataclass(frozen=True) +class SessionTranscript(WireModel): + assignment_id: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + started_at: str + ended_at: str + event_count: int + events: tuple[TranscriptEvent, ...] + schema_version: str = TRANSCRIPT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SessionTranscript: + version = _string(data, "schema_version") + if version != TRANSCRIPT_SCHEMA_VERSION: + raise ProtocolError(f"unsupported transcript schema version {version!r}") + events = tuple( + TranscriptEvent.from_wire(item) for item in _object_list(data, "events") + ) + event_count = _nonnegative_integer(data, "event_count") + if event_count != len(events): + raise ProtocolError( + f"event_count is {event_count}, but transcript contains {len(events)} events" + ) + return cls( + assignment_id=_string(data, "assignment_id"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + started_at=_string(data, "started_at"), + ended_at=_string(data, "ended_at"), + event_count=event_count, + events=events, + ) + + def events_of_type(self, event_type: str) -> tuple[TranscriptEvent, ...]: + return tuple(event for event in self.events if event.event_type == event_type) + + def count(self, event_type: str) -> int: + return sum(event.event_type == event_type for event in self.events) + + +@dataclass(frozen=True) +class EvalSelection(WireModel): + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> EvalSelection: + return cls(_string(data, "eval_key"), _string(data, "eval_version")) + + +@dataclass(frozen=True) +class SkippedEval(WireModel): + eval_key: str + eval_version: str + reason_code: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SkippedEval: + return cls( + _string(data, "eval_key"), + _string(data, "eval_version"), + _string(data, "reason_code"), + ) + + +@dataclass(frozen=True) +class PlanRequest(WireModel): + worker_id: str + lease_generation: int + selected: tuple[EvalSelection, ...] = () + skipped: tuple[SkippedEval, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + selected=tuple( + EvalSelection.from_wire(item) for item in _object_list(data, "selected") + ), + skipped=tuple( + SkippedEval.from_wire(item) for item in _object_list(data, "skipped") + ), + ) + + +@dataclass(frozen=True) +class PlannedRun(WireModel): + evaluation_run_id: str + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + return cls( + _string(data, "evaluation_run_id"), + _string(data, "eval_key"), + _string(data, "eval_version"), + ) + + +@dataclass(frozen=True) +class PlanResponse(WireModel): + assignment_id: str + assignment_status: str + runs: tuple[PlannedRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + assignment_status=_string(data, "assignment_status"), + runs=tuple( + PlannedRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatRun(WireModel): + evaluation_run_id: str + state: str + progress: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRun: + progress = data.get("progress") + if progress is not None: + if isinstance(progress, bool) or not isinstance(progress, (int, float)): + raise ProtocolError("progress must be a number or null") + progress = float(progress) + if not math.isfinite(progress) or not 0 <= progress <= 1: + raise ProtocolError("progress must be finite and between 0 and 1") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + state=_string(data, "state"), + progress=progress, + ) + + +@dataclass(frozen=True) +class HeartbeatRequest(WireModel): + worker_id: str + lease_generation: int + runs: tuple[HeartbeatRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + runs=tuple( + HeartbeatRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatResponse(WireModel): + lease_expires_at: str + accepted_run_ids: tuple[str, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + lease_expires_at=_string(data, "lease_expires_at"), + accepted_run_ids=_string_list(data, "accepted_run_ids"), + ) + + +@dataclass(frozen=True) +class ResultItem(WireModel): + result_key: str + result_kind: ResultKind + numeric_value: float | None = None + bool_value: bool | None = None + text_value: str | None = None + unit: str = "" + display_value: str | None = None + description: str | None = None + reasoning: str | None = None + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultItem: + numeric = data.get("numeric_value") + if numeric is not None: + if isinstance(numeric, bool) or not isinstance(numeric, (int, float)): + raise ProtocolError("numeric_value must be a number or null") + numeric = float(numeric) + if not math.isfinite(numeric): + raise ProtocolError("numeric_value must be finite") + boolean = data.get("bool_value") + if boolean is not None and not isinstance(boolean, bool): + raise ProtocolError("bool_value must be a boolean or null") + return cls( + result_key=_string(data, "result_key"), + result_kind=_enum(ResultKind, data, "result_kind"), + numeric_value=numeric, + bool_value=boolean, + text_value=_optional_string(data, "text_value"), + unit=_string(data, "unit"), + display_value=_optional_string(data, "display_value"), + description=_optional_string(data, "description"), + reasoning=_optional_string(data, "reasoning"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class ResultRequest(WireModel): + submission_id: str + worker_id: str + lease_generation: int + status: TerminalRunStatus + started_at: str + finished_at: str + duration_ms: int + summary: str | None + results: tuple[ResultItem, ...] + error_code: str | None + error_message: str | None + protocol_version: str = PROTOCOL_VERSION + result_schema_version: str = RESULT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultRequest: + validate_protocol_version(_string(data, "protocol_version")) + result_schema_version = _string(data, "result_schema_version") + if result_schema_version != RESULT_SCHEMA_VERSION: + raise ProtocolError( + f"unsupported result schema version {result_schema_version!r}" + ) + return cls( + submission_id=_string(data, "submission_id"), + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + status=_enum(TerminalRunStatus, data, "status"), + started_at=_string(data, "started_at"), + finished_at=_string(data, "finished_at"), + duration_ms=_nonnegative_integer(data, "duration_ms"), + summary=_optional_string(data, "summary"), + results=tuple( + ResultItem.from_wire(item) for item in _object_list(data, "results") + ), + error_code=_optional_string(data, "error_code"), + error_message=_optional_string(data, "error_message"), + ) + + +@dataclass(frozen=True) +class ResultResponse(WireModel): + evaluation_run_id: str + submission_id: str + status: str + idempotent_replay: bool + result_count: int + result_checksum: str + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay") + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + submission_id=_string(data, "submission_id"), + status=_string(data, "status"), + idempotent_replay=replay, + result_count=_nonnegative_integer(data, "result_count"), + result_checksum=_string(data, "result_checksum"), + ) + + +@dataclass(frozen=True) +class RemoteError(WireModel): + code: str + message: str + retryable: bool + request_id: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RemoteError: + retryable = data.get("retryable") + if not isinstance(retryable, bool): + raise ProtocolError("retryable must be a boolean") + return cls( + code=_string(data, "code"), + message=_string(data, "message"), + retryable=retryable, + request_id=_string(data, "request_id"), + ) + + +@dataclass(frozen=True) +class ErrorResponse(WireModel): + error: RemoteError + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ErrorResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls(RemoteError.from_wire(_object(data.get("error"), "error"))) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py new file mode 100644 index 00000000..8a21402c --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -0,0 +1,563 @@ +"""Async worker state machine for Evaluator v2.""" + +from __future__ import annotations + +import asyncio +import inspect +import logging +import os +import socket +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from failproofai_sdk import __version__ +from failproofai_sdk.evaluator.authoring import ( + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + MAX_CLAIM_CAPACITY, + MAX_CLAIM_WAIT_SECONDS, + MAX_WORKER_ID_BYTES, + Assignment, + ClaimRequest, + EvalSelection, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + RegisterRequest, + ResultRequest, + SkippedEval, + TerminalRunStatus, +) + +logger = logging.getLogger("failproofai_sdk.evaluator") + + +def _utc_now() -> str: + return ( + datetime.now(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def _positive_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + if value <= 0: + raise ValueError(f"{name} must be greater than zero") + return value + + +@dataclass(frozen=True) +class WorkerConfig: + server_url: str + credential: str + worker_id: str + max_concurrency: int = 1 + claim_wait_seconds: int = 20 + request_timeout_seconds: int = 30 + drain_timeout_seconds: int = 60 + + @classmethod + def from_env(cls) -> WorkerConfig: + server_url = os.environ.get("FAILPROOFAI_EVALUATOR_URL", "").strip() + credential = os.environ.get("FAILPROOFAI_EVALUATOR_TOKEN", "").strip() + if not server_url: + raise ValueError("FAILPROOFAI_EVALUATOR_URL is required") + if not credential: + raise ValueError("FAILPROOFAI_EVALUATOR_TOKEN is required") + worker_id = os.environ.get("FAILPROOFAI_EVALUATOR_WORKER_ID", "").strip() + if not worker_id: + worker_id = f"{socket.gethostname()}-{os.getpid()}" + if len(worker_id.encode("utf-8")) > MAX_WORKER_ID_BYTES: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_WORKER_ID exceeds {MAX_WORKER_ID_BYTES} bytes" + ) + if any(ord(character) < 32 or ord(character) == 127 for character in worker_id): + raise ValueError( + "FAILPROOFAI_EVALUATOR_WORKER_ID must not contain control characters" + ) + config = cls( + server_url=server_url, + credential=credential, + worker_id=worker_id, + max_concurrency=_positive_int("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), + claim_wait_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", 20 + ), + request_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30 + ), + drain_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 + ), + ) + if config.max_concurrency > MAX_CLAIM_CAPACITY: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds {MAX_CLAIM_CAPACITY}" + ) + if config.claim_wait_seconds > MAX_CLAIM_WAIT_SECONDS: + raise ValueError( + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS exceeds " + f"{MAX_CLAIM_WAIT_SECONDS}" + ) + if config.request_timeout_seconds <= config.claim_wait_seconds: + raise ValueError( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS must exceed " + "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS" + ) + return config + + +class WorkerRuntime: + def __init__( + self, + evaluator: Evaluator, + config: WorkerConfig, + *, + client: EvaluatorClient | None = None, + ) -> None: + self.evaluator = evaluator + self.config = config + self.client = client or EvaluatorClient( + base_url=config.server_url, + credential=config.credential, + timeout_seconds=config.request_timeout_seconds, + ) + self._stopping = asyncio.Event() + self._active: set[asyncio.Task[None]] = set() + self._heartbeat_interval = 30 + self._claim_limit = config.max_concurrency + self._lease_duration = 120 + self._disabled_definitions: set[str] = set() + self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + self._registered = False + self._last_server_contact: float | None = None + self._metric_lock = threading.Lock() + self._metrics: dict[str, int] = {} + + async def register(self) -> None: + try: + response = await self._call_client( + self.client.register, + RegisterRequest( + worker_id=self.config.worker_id, + sdk_version=__version__, + catalog_revision=self.evaluator.catalog_revision, + max_concurrency=self.config.max_concurrency, + definitions=self.evaluator.catalog(), + ), + ) + except Exception: + self._increment("registration_failure") + raise + self._heartbeat_interval = response.heartbeat_interval_seconds + self._lease_duration = response.lease_duration_seconds + self._claim_limit = min(self.config.max_concurrency, response.claim_limit) + if ( + self._heartbeat_interval <= 0 + or self._lease_duration <= self._heartbeat_interval + or self._claim_limit <= 0 + ): + self._increment("registration_failure") + raise RuntimeError( + "server returned invalid evaluator timing or claim limits" + ) + self._disabled_definitions = set(response.disabled_definitions) + self._registered = True + self._increment("registration_success") + + async def run_forever(self) -> None: + await self.register() + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + # With a transport error the server may have committed the + # lease while its response was lost. Waiting out that lease is + # what prevents a blind second claim from exceeding capacity. + await self._wait_or_stop( + float(self._lease_duration) if error.status is None else 1.0 + ) + continue + assignments = self._validated_assignments(response.assignments, capacity) + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + + await self.drain() + + async def run_once(self) -> int: + """Claim once and finish the returned assignments; useful for jobs/tests.""" + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=self._claim_limit, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + assignments = self._validated_assignments( + response.assignments, self._claim_limit + ) + self._increment("assignments_claimed", len(assignments)) + await asyncio.gather(*(self.process_assignment(item) for item in assignments)) + return len(assignments) + + def stop(self) -> None: + self._stopping.set() + + async def drain(self) -> None: + self._reap_finished() + if not self._active: + return + done, pending = await asyncio.wait( + self._active, timeout=self.config.drain_timeout_seconds + ) + for task in done: + self._consume_task(task) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + self._active.clear() + + async def process_assignment(self, assignment: Assignment) -> None: + session = await self._call_client( + self.client.transcript, + assignment, + worker_id=self.config.worker_id, + ) + if session.session_revision_id != assignment.session_revision_id: + raise RuntimeError("transcript session revision does not match assignment") + + selected: list[EvalDefinition] = [] + skipped: list[SkippedEval] = [] + for definition in self.evaluator.definitions: + if definition.eval_key in self._disabled_definitions: + skipped.append(self._skipped(definition, "disabled_by_server")) + self._increment("conditions_skipped") + continue + if definition.condition is None: + selected.append(definition) + continue + try: + condition = await self._invoke(definition.condition, session) + if isinstance(condition, ConditionResult): + applicable = condition.applicable + reason_code = condition.reason_code + elif isinstance(condition, bool): + applicable = condition + reason_code = "condition_false" + else: + raise TypeError("condition must return bool or ConditionResult") + except Exception as error: # noqa: BLE001 - isolates customer condition code + logger.warning( + "evaluator condition failed", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + skipped.append(self._skipped(definition, "condition_error")) + self._increment("conditions_skipped") + continue + if applicable: + selected.append(definition) + self._increment("conditions_selected") + else: + skipped.append(self._skipped(definition, reason_code)) + self._increment("conditions_skipped") + + plan = await self._call_client( + self.client.plan, + assignment.assignment_id, + PlanRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + selected=tuple( + EvalSelection(item.eval_key, item.eval_version) for item in selected + ), + skipped=tuple(skipped), + ), + ) + if plan.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned a plan for a different assignment") + expected_status = "planned" if selected else "skipped" + if plan.assignment_status != expected_status: + raise RuntimeError("server returned an inconsistent assignment status") + + definitions = {(item.eval_key, item.eval_version): item for item in selected} + run_definitions: list[tuple[str, EvalDefinition]] = [] + run_ids: set[str] = set() + for run in plan.runs: + if run.evaluation_run_id in run_ids: + raise RuntimeError("server returned a duplicate evaluation run id") + run_ids.add(run.evaluation_run_id) + definition = definitions.pop((run.eval_key, run.eval_version), None) + if definition is None: + raise RuntimeError("server returned an unrequested evaluation run") + run_definitions.append((run.evaluation_run_id, definition)) + if definitions: + raise RuntimeError("server omitted a selected evaluation run") + + tasks = { + run_id: asyncio.create_task( + self._execute_run(assignment, run_id, definition, session) + ) + for run_id, definition in run_definitions + } + heartbeat = asyncio.create_task(self._heartbeat(assignment, tasks)) + try: + outcomes = await asyncio.gather(*tasks.values(), return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise outcome + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _execute_run( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + async with self._eval_semaphore: + await self._execute_run_in_slot(assignment, run_id, definition, session) + + async def _execute_run_in_slot( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + started_at = _utc_now() + started = time.monotonic() + try: + invocation = self._invoke(definition.function, session) + result = ( + await asyncio.wait_for(invocation, timeout=definition.timeout_seconds) + if definition.timeout_seconds is not None + else await invocation + ) + if not isinstance(result, EvalResult): + raise TypeError("evaluation must return EvalResult") + items = result.result_items(definition.eval_key) + if not any( + item.result_key == definition.eval_key + and item.result_kind == definition.result_kind + for item in items + ): + raise ValueError( + "evaluation result does not contain its declared primary result" + ) + status = TerminalRunStatus.SUCCEEDED + summary = result.summary + error_code = None + error_message = None + except asyncio.TimeoutError: + await self._cancel_hook(definition, session) + items = () + status = TerminalRunStatus.TIMED_OUT + summary = None + error_code = "eval_timeout" + error_message = "evaluation exceeded its configured timeout" + except asyncio.CancelledError: + await self._cancel_hook(definition, session) + raise + except Exception as error: # noqa: BLE001 - converts customer eval failures + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + error_message = f"evaluation raised {type(error).__name__}" + + request = ResultRequest( + submission_id=str(uuid4()), + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + status=status, + started_at=started_at, + finished_at=_utc_now(), + duration_ms=max(0, round((time.monotonic() - started) * 1_000)), + summary=summary, + results=items, + error_code=error_code, + error_message=error_message, + ) + await self._call_client(self.client.submit_result, run_id, request) + self._increment(f"runs_{status.value}") + + async def _cancel_hook(self, definition: EvalDefinition, session: Any) -> None: + if definition.on_cancel is None: + return + try: + await self._invoke(definition.on_cancel, session) + except Exception as error: # noqa: BLE001 - cancellation hooks are customer code + logger.warning( + "evaluator cancellation hook failed", + extra={"error_type": type(error).__name__}, + ) + + async def _heartbeat( + self, + assignment: Assignment, + tasks: dict[str, asyncio.Task[None]], + ) -> None: + while True: + await asyncio.sleep(self._heartbeat_interval) + active = tuple( + HeartbeatRun(evaluation_run_id=run_id, state="running") + for run_id, task in tasks.items() + if not task.done() + ) + if not active: + return + try: + response = await self._call_client( + self.client.heartbeat, + HeartbeatRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + runs=active, + ), + ) + accepted = set(response.accepted_run_ids) + for run_id, task in tasks.items(): + if not task.done() and run_id not in accepted: + task.cancel() + except EvaluatorAPIError as error: + if error.code == "lease_lost": + self._increment("leases_lost") + for task in tasks.values(): + task.cancel() + return + logger.warning( + "evaluator heartbeat failed", + extra={ + "assignment_id": assignment.assignment_id, + "code": error.code, + }, + ) + self._increment("heartbeat_failures") + + @staticmethod + async def _invoke(function, session): + if inspect.iscoroutinefunction(function): + return await function(session) + result = await asyncio.to_thread(function, session) + if inspect.isawaitable(result): + return await result + return result + + @staticmethod + def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + def _reap_finished(self) -> None: + done = {task for task in self._active if task.done()} + self._active.difference_update(done) + for task in done: + self._consume_task(task) + + @staticmethod + def _validated_assignments( + assignments: tuple[Assignment, ...], capacity: int + ) -> tuple[Assignment, ...]: + if len(assignments) > capacity: + raise RuntimeError("server returned more assignments than requested") + assignment_ids = [item.assignment_id for item in assignments] + if len(assignment_ids) != len(set(assignment_ids)): + raise RuntimeError("server returned duplicate assignments") + return assignments + + @staticmethod + def _consume_task(task: asyncio.Task[None]) -> None: + try: + task.result() + except asyncio.CancelledError: + pass + except Exception: + logger.exception("evaluator assignment failed") + + async def _wait_for_progress(self) -> None: + if not self._active: + return + stop_task = asyncio.create_task(self._stopping.wait()) + try: + await asyncio.wait( + (*self._active, stop_task), return_when=asyncio.FIRST_COMPLETED + ) + finally: + if not stop_task.done(): + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + + async def _wait_or_stop(self, seconds: float) -> None: + try: + await asyncio.wait_for(self._stopping.wait(), timeout=seconds) + except asyncio.TimeoutError: + pass + + async def _call_client(self, function, *args, **kwargs): + result = await asyncio.to_thread(function, *args, **kwargs) + self._last_server_contact = time.monotonic() + return result + + def _increment(self, name: str, amount: int = 1) -> None: + with self._metric_lock: + self._metrics[name] = self._metrics.get(name, 0) + amount + + def metrics(self) -> dict[str, int]: + with self._metric_lock: + return dict(self._metrics) + + def is_ready(self) -> bool: + if ( + self._stopping.is_set() + or not self._registered + or self._last_server_contact is None + ): + return False + return time.monotonic() - self._last_server_contact <= max( + float(self._lease_duration), 60.0 + ) diff --git a/sdk/python/tests/fixtures/evaluator_v2/README.md b/sdk/python/tests/fixtures/evaluator_v2/README.md new file mode 100644 index 00000000..e57c93bd --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/README.md @@ -0,0 +1,28 @@ +# Evaluator v2 contract fixtures + +`contract.json` is the Checkpoint 0 wire contract shared by the Rust server and +the zero-dependency Python SDK. The matching copy lives at +`server/tests/fixtures/evaluator_v2/contract.json` in the `agenteye` repository. +Change both copies together. + +Contract rules: + +- The only accepted protocol major is the exact string `"2"`. Unsupported + majors return `426 unsupported_protocol_version`. +- Unknown JSON fields are ignored so either side may add optional fields within + major version 2. Removing, renaming, or changing the meaning of a field needs + a new major version. +- Worker payloads never carry authoritative tenant or evaluator-instance + identity. The server derives those from the credential and leased record. +- `lease_generation` is the fencing token. `409 lease_lost` is terminal for the + affected local execution; the SDK must stop heartbeating or submitting it. +- `submission_id` is an idempotency key. Replaying identical content succeeds; + reusing it for different content returns `409 submission_conflict`. +- Transcript overflow is terminal in v2 (`413 transcript_too_large`); the server + never silently truncates the evaluated input. +- Only errors marked `retryable` may be retried automatically. HTTP method alone + is not enough to decide whether a protocol operation is safe to replay. + +The timing and payload limits in the fixture are normative defaults. A register +response may lower the worker's effective concurrency, heartbeat interval, or +lease duration, but may not raise a client-side payload bound. diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json new file mode 100644 index 00000000..aa0cf275 --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -0,0 +1,225 @@ +{ + "fixture_revision": "evaluator-v2-2026-08-28.2", + "protocol": { + "supported_major_versions": ["2"], + "transcript_schema_version": "2", + "result_schema_version": "2" + }, + "http": { + "register": "/v1/evaluator/workers/register", + "claim": "/v1/evaluator/assignments/claim", + "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "plan": "/v1/evaluator/assignments/{assignment_id}/plan", + "heartbeat": "/v1/evaluator/runs/heartbeat", + "result": "/v1/evaluator/runs/{evaluation_run_id}/result", + "worker_id_header": "X-FailproofAI-Worker-Id", + "lease_generation_header": "X-FailproofAI-Lease-Generation" + }, + "timing": { + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "max_claim_wait_seconds": 25, + "max_attempts": 5 + }, + "limits": { + "max_catalog_definitions": 100, + "max_claim_capacity": 32, + "max_transcript_bytes": 26214400, + "max_results_per_run": 25, + "max_eval_key_bytes": 128, + "max_display_name_bytes": 128, + "max_version_bytes": 128, + "max_worker_id_bytes": 128, + "max_label_bytes": 64, + "max_labels_per_result": 20, + "max_summary_bytes": 4096, + "max_reasoning_bytes": 16384, + "max_unit_bytes": 64, + "max_display_value_bytes": 256, + "max_description_bytes": 1000, + "max_error_code_bytes": 64, + "max_error_message_bytes": 4096 + }, + "errors": { + "invalid_credentials": {"http_status": 401, "retryable": false}, + "instance_disabled": {"http_status": 403, "retryable": false}, + "assignment_not_found": {"http_status": 404, "retryable": false}, + "run_not_found": {"http_status": 404, "retryable": false}, + "catalog_mismatch": {"http_status": 409, "retryable": false}, + "lease_lost": {"http_status": 409, "retryable": false}, + "plan_conflict": {"http_status": 409, "retryable": false}, + "submission_conflict": {"http_status": 409, "retryable": false}, + "retry_budget_exhausted": {"http_status": 409, "retryable": false}, + "transcript_too_large": {"http_status": 413, "retryable": false}, + "invalid_request": {"http_status": 422, "retryable": false}, + "invalid_catalog": {"http_status": 422, "retryable": false}, + "unsupported_protocol_version": {"http_status": 426, "retryable": false}, + "internal_error": {"http_status": 500, "retryable": true} + }, + "samples": { + "register_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "sdk_version": "0.0.1b2", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "max_concurrency": 4, + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"] + } + ] + }, + "register_response": { + "protocol_version": "2", + "evaluator_instance_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23101", + "evaluator_kind": "customer", + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "claim_limit": 4, + "disabled_definitions": [] + }, + "claim_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "capacity": 2, + "wait_seconds": 20 + }, + "claim_response": { + "protocol_version": "2", + "assignments": [ + { + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "lease_generation": 3, + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "trigger_reason": "agent_end", + "event_count": 42, + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript" + } + ] + }, + "transcript_response": { + "schema_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "started_at": "2026-08-28T11:58:00.000000Z", + "ended_at": "2026-08-28T12:00:00.000000Z", + "event_count": 2, + "events": [ + { + "id": "evt-tool-1", + "ts": "2026-08-28T11:59:00.000000Z", + "event_type": "tool_use", + "payload": {"tool_name": "search"} + }, + { + "id": "evt-end-1", + "ts": "2026-08-28T12:00:00.000000Z", + "event_type": "agent_end", + "payload": {"summary": "Done"} + } + ] + }, + "plan_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "selected": [ + {"eval_key": "tool_efficiency", "eval_version": "1.2.0"} + ], + "skipped": [ + { + "eval_key": "answer_groundedness", + "eval_version": "2.1.0", + "reason_code": "no_retrieval_events" + } + ] + }, + "plan_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "assignment_status": "planned", + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "eval_key": "tool_efficiency", + "eval_version": "1.2.0" + } + ] + }, + "heartbeat_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "state": "running", + "progress": 0.5 + } + ] + }, + "heartbeat_response": { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": ["018f47a8-7c1d-7e21-a22a-79f7a4d23103"] + }, + "result_request": { + "protocol_version": "2", + "result_schema_version": "2", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "status": "succeeded", + "started_at": "2026-08-28T12:00:10.000000Z", + "finished_at": "2026-08-28T12:00:10.812000Z", + "duration_ms": 812, + "summary": "Used a compact tool set without retries.", + "results": [ + { + "result_key": "tool_efficiency", + "result_kind": "score", + "numeric_value": 0.92, + "bool_value": true, + "text_value": null, + "unit": "ratio", + "display_value": "92%", + "description": "Distinct tools divided by total tool calls", + "reasoning": "3 distinct tools across 3 calls", + "labels": ["tools", "deterministic"] + } + ], + "error_code": null, + "error_message": null + }, + "result_response": { + "protocol_version": "2", + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "status": "committed", + "idempotent_replay": false, + "result_count": 1, + "result_checksum": "sha256:8cbd34f2d95d" + }, + "error_response": { + "protocol_version": "2", + "error": { + "code": "lease_lost", + "message": "The assignment lease is no longer owned by this worker.", + "retryable": false, + "request_id": "req-018f47a8" + } + } + } +} diff --git a/sdk/python/tests/test_evaluator_authoring.py b/sdk/python/tests/test_evaluator_authoring.py new file mode 100644 index 00000000..578b1220 --- /dev/null +++ b/sdk/python/tests/test_evaluator_authoring.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +import math + +import pytest + +from failproofai_sdk.evaluator import ( + Assertion, + EvalResult, + Evaluator, + Metric, + ResultKind, + Score, +) + + +def test_catalog_is_stable_across_registration_order(): + first = Evaluator(name="acme", version="2026.08.1") + second = Evaluator(name="acme", version="2026.08.1") + + @first.eval("zeta_check", version="1", labels=["z", "a"]) + def first_zeta(session): + return EvalResult(score=Score(1)) + + @first.eval("alpha_check", version="1") + def first_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("alpha_check", version="1") + def second_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("zeta_check", version="1", labels=["a", "z"]) + def second_zeta(session): + return EvalResult(score=Score(1)) + + assert first.catalog_revision == second.catalog_revision + assert [item.eval_key for item in first.catalog()] == ["alpha_check", "zeta_check"] + + +def test_duplicate_eval_keys_are_rejected_even_when_versions_differ(): + evaluator = Evaluator(name="acme", version="1") + + @evaluator.eval("quality", version="1") + def quality_v1(session): + return EvalResult(score=Score(1)) + + with pytest.raises(ValueError, match="duplicate eval key"): + + @evaluator.eval("quality", version="2") + def quality_v2(session): + return EvalResult(score=Score(1)) + + +@pytest.mark.parametrize("value", [-0.01, 1.01, math.nan, math.inf]) +def test_scores_are_finite_ratios(value): + with pytest.raises(ValueError): + Score(value) + + +def test_result_presentation_fields_are_bounded_before_networking(): + with pytest.raises(ValueError, match="unit is 65 bytes"): + Metric(1, unit="u" * 65) + with pytest.raises(ValueError, match="display value is 257 bytes"): + Score(1, display_value="x" * 257) + with pytest.raises(ValueError, match="description is 1001 bytes"): + Assertion(True, description="x" * 1001) + + +def test_eval_result_expands_to_typed_long_form_rows(): + result = EvalResult( + score=Score(0.75, passed=True, unit="ratio"), + metrics={"call_count": Metric(4, unit="calls")}, + assertions={"had_output": Assertion(True)}, + reasoning="Three useful calls out of four.", + labels=("tools",), + ) + + items = result.result_items("tool_efficiency") + assert [item.result_kind for item in items] == [ + ResultKind.SCORE, + ResultKind.METRIC, + ResultKind.ASSERTION, + ] + assert items[0].reasoning == "Three useful calls out of four." + assert items[1].numeric_value == 4 + assert items[2].bool_value is True + + +def test_empty_eval_result_is_rejected_when_serialized(): + with pytest.raises(ValueError, match="must contain"): + EvalResult().result_items("quality") + + +def test_result_keys_must_be_unique_across_kinds(): + result = EvalResult(score=Score(1), metrics={"quality": 1}) + with pytest.raises(ValueError, match="result keys must be unique"): + result.result_items("quality") + + +def test_sync_and_async_functions_share_one_call_path(): + async def async_eval(session): + return EvalResult(score=Score(1)) + + def sync_eval(session): + return EvalResult(score=Score(0.5)) + + async def exercise(): + sync_result = await Evaluator.call(sync_eval, None) + async_result = await Evaluator.call(async_eval, None) + return sync_result, async_result + + sync_result, async_result = asyncio.run(exercise()) + assert sync_result.score.value == 0.5 + assert async_result.score.value == 1 + + +def test_keys_are_machine_safe_and_versions_are_explicit(): + evaluator = Evaluator(name="acme", version="1") + with pytest.raises(ValueError, match="must match"): + evaluator.eval("Not Safe", version="1") + with pytest.raises(ValueError, match="must not be empty"): + evaluator.eval("safe", version="") diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py new file mode 100644 index 00000000..7513e306 --- /dev/null +++ b/sdk/python/tests/test_evaluator_client.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError, URLError + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvaluatorAPIError, + EvaluatorClient, + ResultRequest, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class Response: + def __init__(self, body): + self.body = json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def read(self, amount): + return self.body[:amount] + + +class RawResponse(Response): + def __init__(self, body: bytes): + self.body = body + + +def test_claim_sends_bearer_auth_and_does_not_retry(): + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + raise URLError("offline") + + client = EvaluatorClient( + base_url="https://cloud.example/api/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + with pytest.raises(EvaluatorAPIError, match="transport_error"): + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + + assert len(calls) == 1 + request, timeout = calls[0] + assert request.full_url == "https://cloud.example/v1/evaluator/assignments/claim" + assert request.get_header("Authorization") == "Bearer secret" + assert timeout == 30 + + +def test_idempotent_result_submission_retries_transport_failure(): + samples = _samples() + calls = 0 + + def opener(request, timeout): + nonlocal calls + calls += 1 + if calls == 1: + raise URLError("reset") + return Response(samples["result_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + response = client.submit_result( + samples["result_response"]["evaluation_run_id"], + ResultRequest.from_wire(samples["result_request"]), + ) + assert response.status == "committed" + assert calls == 2 + + +def test_transcript_url_cannot_exfiltrate_the_worker_credential(): + sample = _samples()["claim_response"]["assignments"][0] + assignment = Assignment.from_wire( + {**sample, "transcript_url": "https://evil.test/read"} + ) + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda *_args, **_kwargs: pytest.fail("network must not be reached"), + ) + with pytest.raises(EvaluatorAPIError, match="outside the configured API origin"): + client.transcript(assignment, worker_id="worker") + + +def test_machine_error_envelope_controls_retryability(): + body = json.dumps(_samples()["error_response"]).encode() + + def opener(request, timeout): + raise HTTPError(request.full_url, 409, "Conflict", {}, io.BytesIO(body)) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == "lease_lost" + assert caught.value.status == 409 + assert caught.value.retryable is False + assert caught.value.request_id == "req-018f47a8" + + +@pytest.mark.parametrize( + ("body", "code"), + [ + (b"not-json", "invalid_response"), + (b"[]", "invalid_response"), + ( + b"{" + b'"padding":"' + b"x" * (2 * 1024 * 1024) + b'"}', + "response_too_large", + ), + ], +) +def test_malformed_or_oversized_server_responses_fail_closed(body, code): + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda request, timeout: RawResponse(body), + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == code + assert caught.value.retryable is False + + +def test_transcript_identity_is_sent_as_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["transcript_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + client.transcript(assignment, worker_id="worker-7") + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_constructor_rejects_unsafe_or_incomplete_configuration(): + with pytest.raises(ValueError, match="absolute"): + EvaluatorClient(base_url="localhost:8080", credential="secret") + with pytest.raises(ValueError, match="credential"): + EvaluatorClient(base_url="https://cloud.example", credential="") + with pytest.raises(ValueError, match="control characters"): + EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") + + +def test_protocol_redirect_does_not_forward_the_bearer_credential(): + exfiltration_attempts = [] + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + exfiltration_attempts.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): + return + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + sink_thread = threading.Thread(target=sink.serve_forever, daemon=True) + sink_thread.start() + + location = f"http://127.0.0.1:{sink.server_address[1]}/steal" + + class Redirector(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(307) + self.send_header("Location", location) + self.end_headers() + + def log_message(self, format, *args): + return + + redirector = ThreadingHTTPServer(("127.0.0.1", 0), Redirector) + redirector_thread = threading.Thread(target=redirector.serve_forever, daemon=True) + redirector_thread.start() + try: + client = EvaluatorClient( + base_url=f"http://127.0.0.1:{redirector.server_address[1]}", + credential="must-not-leak", + max_retries=0, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 0)) + assert caught.value.status == 307 + assert exfiltration_attempts == [] + finally: + redirector.shutdown() + redirector.server_close() + redirector_thread.join(timeout=5) + sink.shutdown() + sink.server_close() + sink_thread.join(timeout=5) diff --git a/sdk/python/tests/test_evaluator_example.py b/sdk/python/tests/test_evaluator_example.py new file mode 100644 index 00000000..e85a545e --- /dev/null +++ b/sdk/python/tests/test_evaluator_example.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ConditionResult + + +def _example_module(): + path = Path(__file__).parents[1] / "examples" / "evaluator_worker.py" + spec = importlib.util.spec_from_file_location("evaluator_worker_example", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_example_registers_deterministic_and_async_evals(monkeypatch): + monkeypatch.delenv("EXAMPLE_JUDGE_URL", raising=False) + module = _example_module() + definitions = {item.eval_key: item for item in module.app.definitions} + assert set(definitions) == {"answer_relevance", "tool_efficiency"} + assert definitions["answer_relevance"].eval_version == "judge-api-v1" + + skipped = definitions["answer_relevance"].condition(None) + assert skipped == ConditionResult(False, "judge_not_configured") + + +def test_example_rejects_non_http_judge_urls(monkeypatch): + module = _example_module() + monkeypatch.setenv("EXAMPLE_JUDGE_URL", "file:///etc/passwd") + with pytest.raises(ValueError, match="absolute http"): + module._call_judge("question", "answer") diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py new file mode 100644 index 00000000..475e0bf3 --- /dev/null +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -0,0 +1,631 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit +from uuid import UUID, uuid5 + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvalResult, + EvalSelection, + Evaluator, + EvaluatorAPIError, + EvaluatorClient, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + ResultItem, + ResultKind, + ResultRequest, + Score, + TerminalRunStatus, + WorkerConfig, + WorkerRuntime, +) + +_NAMESPACE = UUID("4d592d9c-aed4-4f07-9b2d-e14963399df6") + + +class ProtocolState: + def __init__(self) -> None: + self.lock = threading.Lock() + self.base_url = "" + self.instances = { + "customer-a-token": ("instance-customer-a", "customer", "org-a"), + "customer-b-token": ("instance-customer-b", "customer", "org-b"), + "managed-token": ("instance-managed", "managed", None), + } + self.registrations: dict[str, dict] = {} + self.assignments: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + self.result_attempts = 0 + self.result_commits = 0 + self.last_result_body: dict | None = None + self.drop_first_result_response = False + + def add_assignment(self, name: str, *, token: str, org: str) -> str: + assignment_id = str(uuid5(_NAMESPACE, name)) + self.assignments[assignment_id] = { + "token": token, + "org": org, + "status": "available", + "worker_id": None, + "lease_generation": 0, + "expired": False, + "session_id": f"session-{name}", + "session_revision_id": f"revision-{name}", + } + return assignment_id + + def expire(self, assignment_id: str) -> None: + with self.lock: + self.assignments[assignment_id]["expired"] = True + + +class ProtocolServer: + def __init__(self, state: ProtocolState) -> None: + self.state = state + handler = _handler_for(state) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.server.daemon_threads = True + state.base_url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> ProtocolServer: # noqa: PYI034 - Python 3.10 lacks Self + self.thread.start() + return self + + def __exit__(self, *_args) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +def _handler_for(state: ProtocolState): + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + body = self._body() + path = urlsplit(self.path).path + if path == "/v1/evaluator/workers/register": + state.registrations[token] = body + instance_id, kind, _org = state.instances[token] + self._json( + 200, + { + "protocol_version": "2", + "evaluator_instance_id": instance_id, + "evaluator_kind": kind, + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "claim_limit": body["max_concurrency"], + "disabled_definitions": [], + }, + ) + return + if path == "/v1/evaluator/assignments/claim": + self._claim(token, body) + return + if path.endswith("/plan"): + self._plan(token, path.split("/")[-2], body) + return + if path == "/v1/evaluator/runs/heartbeat": + self._heartbeat(token, body) + return + if path.endswith("/result"): + self._result(token, path.split("/")[-2], body) + return + self._error(404, "assignment_not_found", False) + + def do_GET(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + path = urlsplit(self.path).path + if path.endswith("/transcript"): + self._transcript(token, path.split("/")[-2]) + return + self._error(404, "assignment_not_found", False) + + def _claim(self, token: str, body: dict) -> None: + claimed = [] + with state.lock: + for assignment_id, item in state.assignments.items(): + if len(claimed) >= body["capacity"]: + break + if item["token"] != token: + continue + if item["status"] in {"leased", "planned"} and not item["expired"]: + continue + if item["status"] not in {"available", "leased", "planned"}: + continue + item["status"] = "leased" + item["expired"] = False + item["worker_id"] = body["worker_id"] + item["lease_generation"] += 1 + claimed.append(self._assignment_wire(assignment_id, item)) + self._json(200, {"protocol_version": "2", "assignments": claimed}) + + def _transcript(self, token: str, assignment_id: str) -> None: + item = self._leased_assignment( + token, + assignment_id, + self.headers.get("X-FailproofAI-Worker-Id"), + self.headers.get("X-FailproofAI-Lease-Generation"), + ) + if item is None: + return + self._json( + 200, + { + "schema_version": "2", + "assignment_id": assignment_id, + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "started_at": "2026-08-28T12:00:00.000000Z", + "ended_at": "2026-08-28T12:00:01.000000Z", + "event_count": 1, + "events": [ + { + "id": "event-1", + "ts": "2026-08-28T12:00:00.500000Z", + "event_type": "model_response", + "payload": {"content": "done"}, + } + ], + }, + ) + + def _plan(self, token: str, assignment_id: str, body: dict) -> None: + item = self._leased_assignment( + token, assignment_id, body["worker_id"], body["lease_generation"] + ) + if item is None: + return + runs = [] + with state.lock: + for selected in body["selected"]: + run_id = str( + uuid5( + _NAMESPACE, + f"{assignment_id}:{selected['eval_key']}:{selected['eval_version']}", + ) + ) + run = state.runs.setdefault( + run_id, + { + "token": token, + "assignment_id": assignment_id, + "worker_id": body["worker_id"], + "lease_generation": body["lease_generation"], + "submission_id": None, + "checksum": None, + }, + ) + if run["submission_id"] is None: + run["worker_id"] = body["worker_id"] + run["lease_generation"] = body["lease_generation"] + runs.append({"evaluation_run_id": run_id, **selected}) + item["status"] = "planned" if runs else "skipped" + self._json( + 200, + { + "protocol_version": "2", + "assignment_id": assignment_id, + "assignment_status": item["status"], + "runs": runs, + }, + ) + + def _heartbeat(self, token: str, body: dict) -> None: + accepted = [] + with state.lock: + for requested in body["runs"]: + run = state.runs.get(requested["evaluation_run_id"]) + assignment = ( + state.assignments.get(run["assignment_id"]) + if run is not None + else None + ) + if ( + run is not None + and assignment is not None + and run["token"] == token + and run["worker_id"] == body["worker_id"] + and run["lease_generation"] == body["lease_generation"] + and assignment["worker_id"] == body["worker_id"] + and assignment["lease_generation"] == body["lease_generation"] + and not assignment["expired"] + ): + accepted.append(requested["evaluation_run_id"]) + if not accepted: + self._error(409, "lease_lost", False) + return + self._json( + 200, + { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": accepted, + }, + ) + + def _result(self, token: str, run_id: str, body: dict) -> None: + with state.lock: + run = state.runs.get(run_id) + if run is None or run["token"] != token: + self._error(404, "run_not_found", False) + return + if ( + run["worker_id"] != body["worker_id"] + or run["lease_generation"] != body["lease_generation"] + ): + self._error(409, "lease_lost", False) + return + checksum = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if run["submission_id"] is not None: + if ( + run["submission_id"] != body["submission_id"] + or run["checksum"] != checksum + ): + self._error(409, "submission_conflict", False) + return + replay = True + else: + run["submission_id"] = body["submission_id"] + run["checksum"] = checksum + state.result_commits += 1 + state.last_result_body = body + replay = False + state.result_attempts += 1 + drop = state.drop_first_result_response and state.result_attempts == 1 + if drop: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self._json( + 200, + { + "protocol_version": "2", + "evaluation_run_id": run_id, + "submission_id": body["submission_id"], + "status": "committed", + "idempotent_replay": replay, + "result_count": len(body["results"]), + "result_checksum": checksum, + }, + ) + + def _leased_assignment( + self, + token: str, + assignment_id: str, + worker_id: str | None, + generation: str | int | None, + ) -> dict | None: + with state.lock: + item = state.assignments.get(assignment_id) + if item is None or item["token"] != token: + self._error(404, "assignment_not_found", False) + return None + try: + generation = int(generation) if generation is not None else None + except ValueError: + generation = None + if ( + item["status"] != "leased" + or item["expired"] + or item["worker_id"] != worker_id + or item["lease_generation"] != generation + ): + self._error(409, "lease_lost", False) + return None + return item + + def _assignment_wire(self, assignment_id: str, item: dict) -> dict: + return { + "assignment_id": assignment_id, + "lease_generation": item["lease_generation"], + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "trigger_reason": "agent_end", + "event_count": 1, + "transcript_url": ( + f"{state.base_url}/v1/evaluator/assignments/" + f"{assignment_id}/transcript" + ), + } + + def _token(self) -> str | None: + value = self.headers.get("Authorization", "") + return ( + value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + ) + + def _body(self) -> dict: + size = int(self.headers.get("Content-Length", "0")) + return json.loads(self.rfile.read(size)) + + def _error(self, status: int, code: str, retryable: bool) -> None: + self._json( + status, + { + "protocol_version": "2", + "error": { + "code": code, + "message": code.replace("_", " "), + "retryable": retryable, + "request_id": "request-e2e", + }, + }, + ) + + def _json(self, status: int, value: dict) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: + return + + return Handler + + +def _client(state: ProtocolState, token: str) -> EvaluatorClient: + return EvaluatorClient( + base_url=state.base_url, + credential=token, + max_retries=2, + sleeper=lambda _seconds: None, + ) + + +def _claim(client: EvaluatorClient, worker_id: str): + return client.claim( + ClaimRequest( + worker_id=worker_id, + catalog_revision="sha256:" + "a" * 64, + capacity=1, + wait_seconds=0, + ) + ) + + +def test_real_http_worker_survives_lost_result_response_without_duplicate_commit(): + state = ProtocolState() + state.add_assignment("runtime", token="customer-a-token", org="org-a") + state.drop_first_result_response = True + evaluator = Evaluator(name="e2e", version="1") + + @evaluator.eval("completion_present", version="1") + def completion_present(session): + return EvalResult(score=Score(float(bool(session.events)))) + + with ProtocolServer(state): + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url=state.base_url, + credential="customer-a-token", + worker_id="worker-a", + claim_wait_seconds=1, + ), + client=_client(state, "customer-a-token"), + ) + + async def run(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(run()) == 1 + + run_id = next(iter(state.runs)) + committed = ResultRequest.from_wire(state.last_result_body) + replay = runtime.client.submit_result(run_id, committed) + assert replay.idempotent_replay is True + + with pytest.raises(EvaluatorAPIError) as caught: + runtime.client.submit_result( + run_id, + replace(committed, summary="different content"), + ) + assert caught.value.code == "submission_conflict" + + assert state.result_attempts == 3 + assert state.result_commits == 1 + assert len(state.runs) == 1 + assert next(iter(state.runs.values()))["submission_id"] is not None + assert runtime.metrics()["runs_succeeded"] == 1 + + +def test_two_workers_racing_receive_one_unique_lease(): + state = ProtocolState() + assignment_id = state.add_assignment("race", token="customer-a-token", org="org-a") + with ProtocolServer(state): + first = _client(state, "customer-a-token") + second = _client(state, "customer-a-token") + with ThreadPoolExecutor(max_workers=2) as executor: + responses = list( + executor.map( + lambda pair: _claim(*pair), + [(first, "worker-a"), (second, "worker-b")], + ) + ) + + claimed = [item for response in responses for item in response.assignments] + assert [item.assignment_id for item in claimed] == [assignment_id] + assert state.assignments[assignment_id]["lease_generation"] == 1 + + +def test_expired_lease_is_reclaimed_and_stale_worker_is_fenced(): + state = ProtocolState() + assignment_id = state.add_assignment( + "reclaim", token="customer-a-token", org="org-a" + ) + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + + assert second.lease_generation == first.lease_generation + 1 + with pytest.raises(EvaluatorAPIError) as caught: + client.transcript(first, worker_id="worker-a") + assert caught.value.code == "lease_lost" + assert caught.value.retryable is False + + with pytest.raises(EvaluatorAPIError) as caught: + client.heartbeat( + HeartbeatRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + runs=(HeartbeatRun(plan.runs[0].evaluation_run_id, "running"),), + ) + ) + assert caught.value.code == "lease_lost" + + +def test_replacement_worker_finishes_after_forced_worker_loss(): + state = ProtocolState() + assignment_id = state.add_assignment( + "forced-worker-loss", token="customer-a-token", org="org-a" + ) + result_sample = ResultRequest( + submission_id=str(uuid5(_NAMESPACE, "forced-worker-loss-result")), + worker_id="worker-a", + lease_generation=1, + status=TerminalRunStatus.SUCCEEDED, + started_at="2026-08-28T12:00:10.000000Z", + finished_at="2026-08-28T12:00:10.100000Z", + duration_ms=100, + summary="Replacement worker completed the evaluation.", + results=( + ResultItem( + result_key="quality", + result_kind=ResultKind.SCORE, + numeric_value=1.0, + ), + ), + error_code=None, + error_message=None, + ) + + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + first_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + + # Worker A disappears after planning. Its lease expires and worker B + # reclaims the same logical assignment and deterministic run. + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + second_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-b", + lease_generation=second.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + assert ( + second_plan.runs[0].evaluation_run_id + == first_plan.runs[0].evaluation_run_id + ) + + stale_result = replace( + result_sample, + worker_id="worker-a", + lease_generation=first.lease_generation, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.submit_result(first_plan.runs[0].evaluation_run_id, stale_result) + assert caught.value.code == "lease_lost" + + replacement_result = replace( + result_sample, + worker_id="worker-b", + lease_generation=second.lease_generation, + ) + committed = client.submit_result( + second_plan.runs[0].evaluation_run_id, replacement_result + ) + assert committed.status == "committed" + assert committed.idempotent_replay is False + + assert state.result_attempts == 1 + assert state.result_commits == 1 + + +def test_customer_tenants_are_isolated_and_managed_worker_coexists(): + state = ProtocolState() + customer_id = state.add_assignment( + "customer", token="customer-a-token", org="org-a" + ) + managed_id = state.add_assignment("managed", token="managed-token", org="org-b") + with ProtocolServer(state): + customer_a = _client(state, "customer-a-token") + customer_b = _client(state, "customer-b-token") + managed = _client(state, "managed-token") + + customer_assignment = _claim(customer_a, "worker-a").assignments[0] + assert customer_assignment.assignment_id == customer_id + assert _claim(customer_b, "worker-b").assignments == () + assert ( + _claim(managed, "worker-managed").assignments[0].assignment_id == managed_id + ) + + stolen = Assignment( + assignment_id=customer_assignment.assignment_id, + lease_generation=customer_assignment.lease_generation, + lease_expires_at=customer_assignment.lease_expires_at, + session_id=customer_assignment.session_id, + session_revision_id=customer_assignment.session_revision_id, + agent_id=customer_assignment.agent_id, + environment=customer_assignment.environment, + trigger_reason=customer_assignment.trigger_reason, + event_count=customer_assignment.event_count, + transcript_url=customer_assignment.transcript_url, + ) + with pytest.raises(EvaluatorAPIError) as caught: + customer_b.transcript(stolen, worker_id="worker-a") + assert caught.value.status == 404 + assert caught.value.code == "assignment_not_found" diff --git a/sdk/python/tests/test_evaluator_main.py b/sdk/python/tests/test_evaluator_main.py new file mode 100644 index 00000000..c846abdc --- /dev/null +++ b/sdk/python/tests/test_evaluator_main.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sys + +import pytest + +from failproofai_sdk.evaluator import Evaluator +from failproofai_sdk.evaluator.__main__ import load_evaluator + + +def test_module_loader_defaults_to_app(tmp_path, monkeypatch): + (tmp_path / "my_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "app = Evaluator(name='example', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("my_evals") + finally: + sys.modules.pop("my_evals", None) + assert isinstance(loaded, Evaluator) + assert loaded.name == "example" + + +def test_module_loader_supports_an_explicit_attribute(tmp_path, monkeypatch): + (tmp_path / "custom_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "worker = Evaluator(name='custom', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("custom_evals:worker") + finally: + sys.modules.pop("custom_evals", None) + assert loaded.name == "custom" + + +def test_module_loader_rejects_the_wrong_object_type(tmp_path, monkeypatch): + (tmp_path / "not_evals.py").write_text("app = object()\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + with pytest.raises(TypeError, match="not Evaluator"): + load_evaluator("not_evals") + finally: + sys.modules.pop("not_evals", None) diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py new file mode 100644 index 00000000..a7fd18ee --- /dev/null +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + UnsupportedProtocolVersion, + protocol, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _contract(): + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("sample", "model"), + [ + ("register_request", RegisterRequest), + ("register_response", RegisterResponse), + ("claim_request", ClaimRequest), + ("claim_response", ClaimResponse), + ("transcript_response", SessionTranscript), + ("plan_request", PlanRequest), + ("plan_response", PlanResponse), + ("heartbeat_request", HeartbeatRequest), + ("heartbeat_response", HeartbeatResponse), + ("result_request", ResultRequest), + ("result_response", ResultResponse), + ("error_response", ErrorResponse), + ], +) +def test_golden_messages_round_trip(sample, model): + wire = _contract()["samples"][sample] + assert model.from_wire(wire).to_wire() == wire + + +def test_unknown_additive_fields_are_tolerated(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["future_optional_field"] = True + assert ClaimRequest.from_wire(wire).capacity == 2 + + +def test_unsupported_major_version_fails_loudly(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["protocol_version"] = "3" + with pytest.raises( + UnsupportedProtocolVersion, match="supported major version is 2" + ): + ClaimRequest.from_wire(wire) + + +def test_transcript_event_count_is_an_integrity_check(): + wire = dict(_contract()["samples"]["transcript_response"]) + wire["event_count"] = 99 + with pytest.raises(ProtocolError, match="transcript contains 2 events"): + SessionTranscript.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "path", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + ("assignments", 0), + 42, + r"assignments\[0\] must be an object", + ), + ( + "register_response", + RegisterResponse, + ("disabled_definitions",), + [42], + r"disabled_definitions\[0\] must be a string", + ), + ( + "heartbeat_response", + HeartbeatResponse, + ("accepted_run_ids", 0), + None, + r"accepted_run_ids\[0\] must be a string", + ), + ( + "plan_response", + PlanResponse, + ("runs", 0), + "not-an-object", + r"runs\[0\] must be an object", + ), + ( + "result_request", + ResultRequest, + ("results", 0, "labels", 0), + 7, + r"labels\[0\] must be a string", + ), + ( + "register_request", + RegisterRequest, + ("definitions", 0, "result_kind"), + "unknown", + "result_kind must be one of", + ), + ], +) +def test_nested_wire_values_fail_with_protocol_errors( + sample, model, path, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + target = wire + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "field", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + "lease_generation", + 0, + "lease_generation must be greater than zero", + ), + ( + "claim_response", + ClaimResponse, + "event_count", + -1, + "event_count must not be negative", + ), + ( + "result_response", + ResultResponse, + "result_count", + -1, + "result_count must not be negative", + ), + ], +) +def test_server_response_counters_and_generations_are_bounded( + sample, model, field, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + if sample == "claim_response": + wire["assignments"][0][field] = value + else: + wire[field] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +def test_fixture_constants_match_the_sdk_contract(): + contract = _contract() + assert contract["protocol"] == { + "supported_major_versions": [protocol.PROTOCOL_VERSION], + "transcript_schema_version": protocol.TRANSCRIPT_SCHEMA_VERSION, + "result_schema_version": protocol.RESULT_SCHEMA_VERSION, + } + assert contract["http"] == { + "register": protocol.REGISTER_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "plan": protocol.PLAN_PATH, + "heartbeat": protocol.HEARTBEAT_PATH, + "result": protocol.RESULT_PATH, + "worker_id_header": protocol.WORKER_ID_HEADER, + "lease_generation_header": protocol.LEASE_GENERATION_HEADER, + } + assert contract["timing"] == { + "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS, + "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS, + "max_claim_wait_seconds": protocol.MAX_CLAIM_WAIT_SECONDS, + "max_attempts": protocol.MAX_ATTEMPTS, + } + assert contract["limits"] == { + "max_catalog_definitions": protocol.MAX_CATALOG_DEFINITIONS, + "max_claim_capacity": protocol.MAX_CLAIM_CAPACITY, + "max_transcript_bytes": protocol.MAX_TRANSCRIPT_BYTES, + "max_results_per_run": protocol.MAX_RESULTS_PER_RUN, + "max_eval_key_bytes": protocol.MAX_EVAL_KEY_BYTES, + "max_display_name_bytes": protocol.MAX_DISPLAY_NAME_BYTES, + "max_version_bytes": protocol.MAX_VERSION_BYTES, + "max_worker_id_bytes": protocol.MAX_WORKER_ID_BYTES, + "max_label_bytes": protocol.MAX_LABEL_BYTES, + "max_labels_per_result": protocol.MAX_LABELS_PER_RESULT, + "max_summary_bytes": protocol.MAX_SUMMARY_BYTES, + "max_reasoning_bytes": protocol.MAX_REASONING_BYTES, + "max_unit_bytes": protocol.MAX_UNIT_BYTES, + "max_display_value_bytes": protocol.MAX_DISPLAY_VALUE_BYTES, + "max_description_bytes": protocol.MAX_DESCRIPTION_BYTES, + "max_error_code_bytes": protocol.MAX_ERROR_CODE_BYTES, + "max_error_message_bytes": protocol.MAX_ERROR_MESSAGE_BYTES, + } + assert contract["errors"] == protocol.ERROR_SPECS + + +def test_session_helpers_use_the_protocol_event_vocabulary(): + session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"]) + assert session.count("tool_use") == 1 + assert session.events_of_type("agent_end")[0].payload["summary"] == "Done" diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py new file mode 100644 index 00000000..2862cfb4 --- /dev/null +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -0,0 +1,701 @@ +from __future__ import annotations + +import asyncio +import json +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimResponse, + ConditionResult, + EvalResult, + Evaluator, + EvaluatorAPIError, + HeartbeatResponse, + PlannedRun, + PlanResponse, + RegisterResponse, + ResultKind, + Score, + SessionTranscript, + WorkerConfig, + WorkerRuntime, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class FakeClient: + def __init__(self): + samples = _samples() + self.assignment = ClaimResponse.from_wire( + samples["claim_response"] + ).assignments[0] + self.session = SessionTranscript.from_wire(samples["transcript_response"]) + self.register_requests = [] + self.claim_requests = [] + self.plans = [] + self.submissions = [] + self.heartbeats = [] + + def register(self, request): + self.register_requests.append(request) + return RegisterResponse.from_wire(_samples()["register_response"]) + + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=(self.assignment,)) + + def transcript(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return self.session + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned" if request.selected else "skipped", + runs=tuple( + PlannedRun(f"run-{item.eval_key}", item.eval_key, item.eval_version) + for item in request.selected + ), + ) + + def submit_result(self, run_id, request): + self.submissions.append((run_id, request)) + + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=tuple(item.evaluation_run_id for item in request.runs), + ) + + +def _runtime(evaluator, client): + return WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=2, + claim_wait_seconds=1, + ), + client=client, + ) + + +def test_condition_failures_are_isolated_and_plan_is_declared_first(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("not_applicable", version="1", when=lambda session: False) + def not_applicable(session): + return EvalResult(score=Score(1)) + + def broken_condition(session): + raise RuntimeError("condition exploded") + + @evaluator.eval("broken_condition", version="1", when=broken_condition) + def never_runs(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + assert len(client.plans) == 1 + assert [item.eval_key for item in client.plans[0].selected] == ["selected"] + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("not_applicable", "condition_false"), + ("broken_condition", "condition_error"), + } + assert [run_id for run_id, _ in client.submissions] == ["run-selected"] + + +def test_condition_can_supply_a_stable_skip_reason(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval( + "retrieval_only", + version="1", + when=lambda session: ConditionResult(False, "no_retrieval_events"), + ) + def retrieval_only(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans[0].skipped[0].reason_code == "no_retrieval_events" + + +def test_one_eval_failure_does_not_block_another_result(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("fails", version="1") + def fails(session): + raise RuntimeError("secret details should be bounded") + + @evaluator.eval("succeeds", version="1") + async def succeeds(session): + await asyncio.sleep(0) + return EvalResult(score=Score(0.8), summary="good") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + by_run = {run_id: request for run_id, request in client.submissions} + assert by_run["run-fails"].status.value == "failed" + assert by_run["run-fails"].error_code == "eval_error" + assert by_run["run-fails"].results == () + assert by_run["run-succeeds"].status.value == "succeeded" + assert by_run["run-succeeds"].results[0].result_kind == ResultKind.SCORE + + +def test_timeout_is_submitted_as_a_terminal_run(): + evaluator = Evaluator(name="test", version="1") + cancelled = [] + + @evaluator.eval( + "slow", + version="1", + timeout_seconds=0.01, + on_cancel=lambda session: cancelled.append(session.session_revision_id), + ) + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert cancelled == [client.assignment.session_revision_id] + + +def test_lost_lease_cancels_local_execution(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1") + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + class LeaseLostClient(FakeClient): + def heartbeat(self, request): + raise EvaluatorAPIError( + status=409, + code="lease_lost", + message="gone", + retryable=False, + ) + + client = LeaseLostClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + with pytest.raises(asyncio.CancelledError): + asyncio.run(runtime.process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_partial_heartbeat_acceptance_cancels_only_the_fenced_run(): + evaluator = Evaluator(name="test", version="1") + + class PartialHeartbeatClient(FakeClient): + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=(request.runs[0].evaluation_run_id,), + ) + + client = PartialHeartbeatClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + first = asyncio.create_task(asyncio.sleep(60)) + second = asyncio.create_task(asyncio.sleep(60)) + heartbeat = asyncio.create_task( + runtime._heartbeat( + client.assignment, {"run-first": first, "run-second": second} + ) + ) + while not client.heartbeats: + await asyncio.sleep(0.001) + for _ in range(100): + if second.done(): + break + await asyncio.sleep(0.001) + assert first.done() is False + assert second.cancelled() is True + heartbeat.cancel() + first.cancel() + await asyncio.gather(first, second, heartbeat, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_transcript_revision_must_match_the_claimed_assignment(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + client.session = SessionTranscript.from_wire( + { + **_samples()["transcript_response"], + "session_revision_id": "different-revision", + } + ) + + with pytest.raises(RuntimeError, match="revision does not match"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans == [] + + +def test_server_cannot_add_a_run_when_every_eval_was_skipped(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + class UnexpectedRunClient(FakeClient): + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-injected", "skipped", "1"),), + ) + + client = UnexpectedRunClient() + with pytest.raises(RuntimeError, match="unrequested evaluation run"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_server_plan_must_match_assignment_and_include_each_selected_eval(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1") + def selected(session): + return EvalResult(score=Score(1)) + + class WrongAssignmentClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id="another-assignment", + assignment_status="planned", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_assignment = WrongAssignmentClient() + with pytest.raises(RuntimeError, match="different assignment"): + asyncio.run( + _runtime(evaluator, wrong_assignment).process_assignment( + wrong_assignment.assignment + ) + ) + + class WrongStatusClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_status = WrongStatusClient() + with pytest.raises(RuntimeError, match="inconsistent assignment status"): + asyncio.run( + _runtime(evaluator, wrong_status).process_assignment( + wrong_status.assignment + ) + ) + + class OmittedRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + ) + + omitted = OmittedRunClient() + with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): + asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + + +def test_server_plan_rejects_duplicate_run_ids(): + evaluator = Evaluator(name="test", version="1") + evaluator.eval("first", version="1")(lambda session: EvalResult(score=Score(1))) + evaluator.eval("second", version="1")(lambda session: EvalResult(score=Score(1))) + + class DuplicateRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun("same-run", "first", "1"), + PlannedRun("same-run", "second", "1"), + ), + ) + + client = DuplicateRunClient() + with pytest.raises(RuntimeError, match="duplicate evaluation run id"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_register_advertises_the_deterministic_catalog(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("quality", version="7") + def quality(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + asyncio.run(runtime.register()) + request = client.register_requests[0] + assert request.catalog_revision == evaluator.catalog_revision + assert request.definitions[0].eval_version == "7" + assert runtime._heartbeat_interval == 30 + + +def test_runtime_readiness_tracks_registration_contact_and_shutdown(monkeypatch): + evaluator = Evaluator(name="test", version="1") + runtime = _runtime(evaluator, FakeClient()) + + assert runtime.is_ready() is False + assert runtime.metrics() == {} + + asyncio.run(runtime.register()) + assert runtime.is_ready() is True + assert runtime.metrics() == {"registration_success": 1} + + last_contact = runtime._last_server_contact + assert last_contact is not None + monkeypatch.setattr(time, "monotonic", lambda: last_contact + 121) + assert runtime.is_ready() is False + + monkeypatch.setattr(time, "monotonic", lambda: last_contact) + runtime.stop() + assert runtime.is_ready() is False + + +def test_runtime_metrics_count_claims_conditions_and_outcomes(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + runtime = _runtime(evaluator, FakeClient()) + + async def exercise(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(exercise()) == 1 + assert runtime.metrics() == { + "assignments_claimed": 1, + "conditions_selected": 1, + "conditions_skipped": 1, + "registration_success": 1, + "runs_succeeded": 1, + } + + +def test_runtime_metrics_count_registration_failure(): + evaluator = Evaluator(name="test", version="1") + + class BrokenClient(FakeClient): + def register(self, request): + raise EvaluatorAPIError( + status=503, + code="unavailable", + message="try later", + retryable=True, + ) + + runtime = _runtime(evaluator, BrokenClient()) + with pytest.raises(EvaluatorAPIError): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_invalid_registration_response_does_not_make_runtime_ready(): + evaluator = Evaluator(name="test", version="1") + + class InvalidTimingClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=120, + lease_duration_seconds=120, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, InvalidTimingClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_lost_claim_response_waits_out_the_lease_before_claiming_again(): + evaluator = Evaluator(name="test", version="1") + + class LostResponseClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=None, + code="transport_error", + message="response lost", + retryable=True, + ) + + runtime = _runtime(evaluator, LostResponseClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [120.0] + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +def test_nonretryable_claim_failure_stops_the_worker(): + evaluator = Evaluator(name="test", version="1") + + class RejectedClaimClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=409, + code="catalog_mismatch", + message="register again with the current catalog", + retryable=False, + ) + + runtime = _runtime(evaluator, RejectedClaimClient()) + with pytest.raises(EvaluatorAPIError, match="catalog_mismatch"): + asyncio.run(runtime.run_forever()) + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +@pytest.mark.parametrize( + ("assignments", "message"), + [ + (lambda item: (item, item), "duplicate assignments"), + ( + lambda item: tuple( + replace(item, assignment_id=f"assignment-{index}") for index in range(3) + ), + "more assignments than requested", + ), + ], +) +def test_claim_response_cannot_exceed_capacity_or_repeat_work(assignments, message): + evaluator = Evaluator(name="test", version="1") + + class InvalidClaimClient(FakeClient): + def claim(self, request): + return ClaimResponse(assignments=assignments(self.assignment)) + + runtime = _runtime(evaluator, InvalidClaimClient()) + with pytest.raises(RuntimeError, match=message): + asyncio.run(runtime.run_once()) + assert runtime.metrics() == {} + + +def test_register_applies_server_claim_limit_and_disabled_definitions(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("disabled", version="1") + def disabled(session): + raise AssertionError("disabled eval must not run") + + class RestrictedClient(FakeClient): + def register(self, request): + self.register_requests.append(request) + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=10, + lease_duration_seconds=120, + claim_limit=1, + disabled_definitions=("disabled",), + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + client = RestrictedClient() + runtime = _runtime(evaluator, client) + + async def exercise(): + await runtime.register() + await runtime.run_once() + + asyncio.run(exercise()) + assert runtime._claim_limit == 1 + assert client.claim_requests[0].capacity == 1 + assert client.plans[0].selected == () + assert client.plans[0].skipped[0].reason_code == "disabled_by_server" + + +def test_worker_config_requires_dedicated_credentials(monkeypatch): + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_URL", raising=False) + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_TOKEN", raising=False) + with pytest.raises(ValueError, match="URL is required"): + WorkerConfig.from_env() + + +def test_worker_config_keeps_long_poll_inside_the_http_timeout(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", "20") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", "20") + with pytest.raises(ValueError, match="must exceed"): + WorkerConfig.from_env() + + +def test_worker_config_rejects_header_control_characters(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_WORKER_ID", "worker\nforged") + with pytest.raises(ValueError, match="control characters"): + WorkerConfig.from_env() + + +def test_graceful_drain_cancels_work_after_the_configured_deadline(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + drain_timeout_seconds=1, + ), + client=client, + ) + cancelled = False + + async def exercise(): + nonlocal cancelled + + async def active_work(): + nonlocal cancelled + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled = True + raise + + task = asyncio.create_task(active_work()) + runtime._active.add(task) + await asyncio.sleep(0) + runtime.config = WorkerConfig( + server_url=runtime.config.server_url, + credential=runtime.config.credential, + worker_id=runtime.config.worker_id, + drain_timeout_seconds=0, + ) + await runtime.drain() + + asyncio.run(exercise()) + assert cancelled is True + assert runtime._active == set() + + +def test_stop_interrupts_capacity_wait_and_enters_drain(): + runtime = _runtime(Evaluator(name="test", version="1"), FakeClient()) + + async def exercise(): + blocker = asyncio.Event() + work = asyncio.create_task(blocker.wait()) + runtime._active.add(work) + await asyncio.sleep(0) + + runtime.stop() + await asyncio.wait_for(runtime._wait_for_progress(), timeout=0.1) + + assert work.done() is False + work.cancel() + await asyncio.gather(work, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_eval_execution_respects_process_concurrency(): + evaluator = Evaluator(name="test", version="1") + active = 0 + peak = 0 + + async def measured(session): + nonlocal active, peak + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) + active -= 1 + return EvalResult(score=Score(1)) + + evaluator.eval("first", version="1")(measured) + evaluator.eval("second", version="1")(measured) + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=1, + ), + client=client, + ) + asyncio.run(runtime.process_assignment(client.assignment)) + assert peak == 1 diff --git a/sdk/python/tests/test_zero_dependencies.py b/sdk/python/tests/test_zero_dependencies.py index a27fd75f..95be7b72 100644 --- a/sdk/python/tests/test_zero_dependencies.py +++ b/sdk/python/tests/test_zero_dependencies.py @@ -300,6 +300,23 @@ def test_importing_the_package_loads_no_framework(): ) +def test_importing_the_package_does_not_load_the_evaluator_runtime(): + """Telemetry-only users do not pay for the separate worker surface.""" + import json + import subprocess + + probe = ( + "import json, sys; import failproofai_sdk; " + "print(json.dumps(sorted(m for m in sys.modules " + "if m.startswith('failproofai_sdk.evaluator'))))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, cwd=str(ROOT), timeout=60 + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip()) == [] + + def test_the_adapter_registry_holds_strings_not_modules(): """`_REGISTRY` maps a name to a dotted path; importing it here would defeat it.""" from failproofai_sdk.integrations import _REGISTRY From 34b99cd395bf55fe360f848238bc5dd783f9e117 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 17:31:11 +0530 Subject: [PATCH 03/20] fix(evaluator): resume only unfinished replayed runs --- sdk/python/failproofai_sdk/evaluator/protocol.py | 5 +++++ sdk/python/failproofai_sdk/evaluator/runtime.py | 2 +- .../tests/fixtures/evaluator_v2/contract.json | 1 + sdk/python/tests/test_evaluator_runtime.py | 15 ++++++++++++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index 09adf8be..beb2391a 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -452,17 +452,22 @@ class PlanResponse(WireModel): assignment_id: str assignment_status: str runs: tuple[PlannedRun, ...] + idempotent_replay: bool = False protocol_version: str = PROTOCOL_VERSION @classmethod def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay", False) + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") return cls( assignment_id=_string(data, "assignment_id"), assignment_status=_string(data, "assignment_status"), runs=tuple( PlannedRun.from_wire(item) for item in _object_list(data, "runs") ), + idempotent_replay=replay, ) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 8a21402c..600713cc 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -334,7 +334,7 @@ async def process_assignment(self, assignment: Assignment) -> None: if definition is None: raise RuntimeError("server returned an unrequested evaluation run") run_definitions.append((run.evaluation_run_id, definition)) - if definitions: + if definitions and not plan.idempotent_replay: raise RuntimeError("server omitted a selected evaluation run") tasks = { diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index aa0cf275..fdc1a0cf 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -150,6 +150,7 @@ "protocol_version": "2", "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", "assignment_status": "planned", + "idempotent_replay": false, "runs": [ { "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index 2862cfb4..bc770e5c 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -285,7 +285,7 @@ def plan(self, assignment_id, request): assert client.submissions == [] -def test_server_plan_must_match_assignment_and_include_each_selected_eval(): +def test_server_plan_must_match_assignment_and_include_each_new_selected_eval(): evaluator = Evaluator(name="test", version="1") @evaluator.eval("selected", version="1") @@ -336,6 +336,19 @@ def plan(self, assignment_id, request): with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + class ReplayedPlanClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + idempotent_replay=True, + ) + + replayed = ReplayedPlanClient() + asyncio.run(_runtime(evaluator, replayed).process_assignment(replayed.assignment)) + assert replayed.submissions == [] + def test_server_plan_rejects_duplicate_run_ids(): evaluator = Evaluator(name="test", version="1") From afa0053a23d11566d5e17ab35a5c281e17439305 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 28 Aug 2026 18:59:03 +0530 Subject: [PATCH 04/20] feat(sdk): run hosted evaluator definitions --- sdk/python/examples/evaluator_worker.py | 10 + .../failproofai_sdk/evaluator/__init__.py | 16 ++ .../failproofai_sdk/evaluator/client.py | 33 ++- .../failproofai_sdk/evaluator/protocol.py | 91 ++++++- .../failproofai_sdk/evaluator/runtime.py | 236 ++++++++++++++---- .../failproofai_sdk/evaluator/source.py | 170 +++++++++++++ .../tests/fixtures/evaluator_v2/contract.json | 30 ++- sdk/python/tests/test_evaluator_client.py | 32 +++ sdk/python/tests/test_evaluator_protocol.py | 7 +- sdk/python/tests/test_evaluator_runtime.py | 133 ++++++++++ sdk/python/tests/test_evaluator_source.py | 60 +++++ 11 files changed, 755 insertions(+), 63 deletions(-) create mode 100644 sdk/python/failproofai_sdk/evaluator/source.py create mode 100644 sdk/python/tests/test_evaluator_source.py diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py index a0cea74b..912b61e5 100644 --- a/sdk/python/examples/evaluator_worker.py +++ b/sdk/python/examples/evaluator_worker.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import ipaddress import json import os from urllib.parse import urlsplit @@ -73,6 +74,15 @@ def _call_judge(question, answer): parsed = urlsplit(url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback: + raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback") token = os.environ.get("EXAMPLE_JUDGE_TOKEN") body = json.dumps({"question": question, "answer": answer}).encode("utf-8") headers = {"Content-Type": "application/json", "Accept": "application/json"} diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py index dff0cad6..a94155ce 100644 --- a/sdk/python/failproofai_sdk/evaluator/__init__.py +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -16,11 +16,13 @@ from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient from failproofai_sdk.evaluator.protocol import ( Assignment, + AssignmentDefinition, CatalogDefinition, ClaimRequest, ClaimResponse, ErrorResponse, EvalSelection, + ExecutionMode, EvaluatorKind, HeartbeatRequest, HeartbeatResponse, @@ -28,6 +30,7 @@ PlannedRun, PlanRequest, PlanResponse, + DefinitionsResponse, ProtocolError, RegisterRequest, RegisterResponse, @@ -43,10 +46,17 @@ UnsupportedProtocolVersion, ) from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime +from failproofai_sdk.evaluator.source import ( + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) __all__ = [ "Assertion", "Assignment", + "AssignmentDefinition", "CatalogDefinition", "ClaimRequest", "ClaimResponse", @@ -55,6 +65,7 @@ "EvalDefinition", "EvalResult", "EvalSelection", + "ExecutionMode", "Evaluator", "EvaluatorAPIError", "EvaluatorClient", @@ -65,6 +76,7 @@ "Metric", "PlanRequest", "PlanResponse", + "DefinitionsResponse", "PlannedRun", "ProtocolError", "RegisterRequest", @@ -82,4 +94,8 @@ "UnsupportedProtocolVersion", "WorkerConfig", "WorkerRuntime", + "UnsafeEvaluatorSource", + "compile_condition", + "compile_evaluator", + "source_checksum", ] diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py index 60fa32c2..fc3c28ce 100644 --- a/sdk/python/failproofai_sdk/evaluator/client.py +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ipaddress import json import random import time @@ -13,6 +14,7 @@ from failproofai_sdk.evaluator.protocol import ( CLAIM_PATH, + DEFINITIONS_PATH, HEARTBEAT_PATH, LEASE_GENERATION_HEADER, MAX_TRANSCRIPT_BYTES, @@ -21,6 +23,7 @@ RESULT_PATH, WORKER_ID_HEADER, Assignment, + DefinitionsResponse, ClaimRequest, ClaimResponse, ErrorResponse, @@ -67,7 +70,11 @@ def __init__( class EvaluatorClient: - """Direct server client; evaluator traffic never passes through the dashboard.""" + """Client for the public Evaluator v2 machine API. + + Hosted workers normally use the FailproofAI dashboard origin. Its ``/v1`` + passthrough forwards this worker's bearer credential to the private server. + """ def __init__( self, @@ -76,12 +83,22 @@ def __init__( credential: str, timeout_seconds: float = 30, max_retries: int = 3, + allow_insecure_http: bool = False, opener: Callable[..., Any] | None = None, sleeper: Callable[[float], None] = time.sleep, ) -> None: parsed = urlsplit(base_url) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("base_url must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback and not allow_insecure_http: + raise ValueError("base_url must use https unless it targets loopback") if not credential or not credential.strip(): raise ValueError("credential must not be empty") if any( @@ -130,6 +147,20 @@ def transcript( ) ) + def definitions( + self, assignment: Assignment, *, worker_id: str + ) -> DefinitionsResponse: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + path = assignment.definitions_url or DEFINITIONS_PATH.format( + assignment_id=assignment.assignment_id + ) + return DefinitionsResponse.from_wire( + self._json("GET", path, None, retry=True, headers=headers) + ) + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: return PlanResponse.from_wire( self._json( diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index beb2391a..e8fa156c 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -15,6 +15,7 @@ REGISTER_PATH = "/v1/evaluator/workers/register" CLAIM_PATH = "/v1/evaluator/assignments/claim" TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +DEFINITIONS_PATH = "/v1/evaluator/assignments/{assignment_id}/definitions" PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" @@ -91,6 +92,11 @@ class ResultKind(str, Enum): ASSERTION = "assertion" +class ExecutionMode(str, Enum): + LOCAL = "local" + PYTHON = "python" + + class TerminalRunStatus(str, Enum): SUCCEEDED = "succeeded" FAILED = "failed" @@ -287,6 +293,7 @@ class Assignment(WireModel): trigger_reason: str event_count: int transcript_url: str + definitions_url: str = "" @classmethod def from_wire(cls, data: Mapping[str, Any]) -> Assignment: @@ -301,6 +308,65 @@ def from_wire(cls, data: Mapping[str, Any]) -> Assignment: trigger_reason=_string(data, "trigger_reason"), event_count=_nonnegative_integer(data, "event_count"), transcript_url=_string(data, "transcript_url"), + definitions_url=str(data.get("definitions_url") or ""), + ) + + +@dataclass(frozen=True) +class AssignmentDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + execution_mode: ExecutionMode = ExecutionMode.LOCAL + condition_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> AssignmentDefinition: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + execution_mode=_enum( + ExecutionMode, + {"execution_mode": data.get("execution_mode") or "local"}, + "execution_mode", + ), + condition_source=_optional_string(data, "condition_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class DefinitionsResponse(WireModel): + assignment_id: str + catalog_revision: str + definitions: tuple[AssignmentDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> DefinitionsResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + catalog_revision=_string(data, "catalog_revision"), + definitions=tuple( + AssignmentDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), ) @@ -437,13 +503,32 @@ class PlannedRun(WireModel): evaluation_run_id: str eval_key: str eval_version: str + execution_mode: ExecutionMode = ExecutionMode.LOCAL + evaluator_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None @classmethod def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") return cls( - _string(data, "evaluation_run_id"), - _string(data, "eval_key"), - _string(data, "eval_version"), + evaluation_run_id=_string(data, "evaluation_run_id"), + eval_key=_string(data, "eval_key"), + eval_version=_string(data, "eval_version"), + execution_mode=_enum( + ExecutionMode, + {"execution_mode": data.get("execution_mode") or "local"}, + "execution_mode", + ), + evaluator_source=_optional_string(data, "evaluator_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, ) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 600713cc..f96d205f 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import concurrent.futures import inspect import logging import os @@ -27,8 +28,10 @@ MAX_CLAIM_WAIT_SECONDS, MAX_WORKER_ID_BYTES, Assignment, + AssignmentDefinition, ClaimRequest, EvalSelection, + ExecutionMode, HeartbeatRequest, HeartbeatRun, PlanRequest, @@ -37,6 +40,11 @@ SkippedEval, TerminalRunStatus, ) +from failproofai_sdk.evaluator.source import ( + compile_condition, + compile_evaluator, + source_checksum, +) logger = logging.getLogger("failproofai_sdk.evaluator") @@ -62,6 +70,18 @@ def _positive_int(name: str, default: int) -> int: return value +def _boolean(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + normalized = raw.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + @dataclass(frozen=True) class WorkerConfig: server_url: str @@ -71,6 +91,7 @@ class WorkerConfig: claim_wait_seconds: int = 20 request_timeout_seconds: int = 30 drain_timeout_seconds: int = 60 + allow_insecure_http: bool = False @classmethod def from_env(cls) -> WorkerConfig: @@ -105,6 +126,9 @@ def from_env(cls) -> WorkerConfig: drain_timeout_seconds=_positive_int( "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 ), + allow_insecure_http=_boolean( + "FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP" + ), ) if config.max_concurrency > MAX_CLAIM_CAPACITY: raise ValueError( @@ -137,6 +161,7 @@ def __init__( base_url=config.server_url, credential=config.credential, timeout_seconds=config.request_timeout_seconds, + allow_insecure_http=config.allow_insecure_http, ) self._stopping = asyncio.Event() self._active: set[asyncio.Task[None]] = set() @@ -145,6 +170,10 @@ def __init__( self._lease_duration = 120 self._disabled_definitions: set[str] = set() self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + self._eval_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=config.max_concurrency, + thread_name_prefix="failproof-eval", + ) self._registered = False self._last_server_contact: float | None = None self._metric_lock = threading.Lock() @@ -183,44 +212,51 @@ async def register(self) -> None: async def run_forever(self) -> None: await self.register() - while not self._stopping.is_set(): - self._reap_finished() - capacity = self._claim_limit - len(self._active) - if capacity <= 0: - await self._wait_for_progress() - continue - try: - response = await self._call_client( - self.client.claim, - ClaimRequest( - worker_id=self.config.worker_id, - catalog_revision=self.evaluator.catalog_revision, - capacity=capacity, - wait_seconds=self.config.claim_wait_seconds, - ), - ) - except EvaluatorAPIError as error: - self._increment("claim_failures") - logger.warning( - "evaluator claim failed", - extra={"code": error.code, "retryable": error.retryable}, - ) - if not error.retryable: - raise - # With a transport error the server may have committed the - # lease while its response was lost. Waiting out that lease is - # what prevents a blind second claim from exceeding capacity. - await self._wait_or_stop( - float(self._lease_duration) if error.status is None else 1.0 + retry_delay = 1.0 + try: + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + wait_seconds=self.config.claim_wait_seconds, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + delay = ( + float(self._lease_duration) + if error.status is None + else retry_delay + ) + await self._wait_or_stop(delay) + retry_delay = min(retry_delay * 2.0, 30.0) + continue + retry_delay = 1.0 + assignments = self._validated_assignments( + response.assignments, capacity ) - continue - assignments = self._validated_assignments(response.assignments, capacity) - for assignment in assignments: - task = asyncio.create_task(self.process_assignment(assignment)) - self._active.add(task) - self._increment("assignments_claimed", len(assignments)) - - await self.drain() + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + finally: + await self.drain() + self._eval_executor.shutdown(wait=False, cancel_futures=True) async def run_once(self) -> int: """Claim once and finish the returned assignments; useful for jobs/tests.""" @@ -267,18 +303,37 @@ async def process_assignment(self, assignment: Assignment) -> None: if session.session_revision_id != assignment.session_revision_id: raise RuntimeError("transcript session revision does not match assignment") - selected: list[EvalDefinition] = [] + descriptors = await self._assignment_definitions(assignment) + selected: list[tuple[AssignmentDefinition, EvalDefinition | None]] = [] skipped: list[SkippedEval] = [] - for definition in self.evaluator.definitions: - if definition.eval_key in self._disabled_definitions: - skipped.append(self._skipped(definition, "disabled_by_server")) + local_definitions = { + (item.eval_key, item.eval_version): item + for item in self.evaluator.definitions + } + for descriptor in descriptors: + local = local_definitions.get( + (descriptor.eval_key, descriptor.eval_version) + ) + if descriptor.execution_mode is ExecutionMode.LOCAL and local is None: + raise RuntimeError("server requested a definition absent from this worker") + if descriptor.eval_key in self._disabled_definitions: + skipped.append(self._skipped_descriptor(descriptor, "disabled_by_server")) self._increment("conditions_skipped") continue - if definition.condition is None: - selected.append(definition) + condition_function = ( + local.condition + if local is not None + else ( + compile_condition(descriptor.condition_source) + if descriptor.condition_source + else None + ) + ) + if condition_function is None: + selected.append((descriptor, local)) continue try: - condition = await self._invoke(definition.condition, session) + condition = await self._invoke(condition_function, session) if isinstance(condition, ConditionResult): applicable = condition.applicable reason_code = condition.reason_code @@ -295,14 +350,14 @@ async def process_assignment(self, assignment: Assignment) -> None: "error_type": type(error).__name__, }, ) - skipped.append(self._skipped(definition, "condition_error")) + skipped.append(self._skipped_descriptor(descriptor, "condition_error")) self._increment("conditions_skipped") continue if applicable: - selected.append(definition) + selected.append((descriptor, local)) self._increment("conditions_selected") else: - skipped.append(self._skipped(definition, reason_code)) + skipped.append(self._skipped_descriptor(descriptor, reason_code)) self._increment("conditions_skipped") plan = await self._call_client( @@ -312,7 +367,8 @@ async def process_assignment(self, assignment: Assignment) -> None: worker_id=self.config.worker_id, lease_generation=assignment.lease_generation, selected=tuple( - EvalSelection(item.eval_key, item.eval_version) for item in selected + EvalSelection(item.eval_key, item.eval_version) + for item, _local in selected ), skipped=tuple(skipped), ), @@ -323,16 +379,50 @@ async def process_assignment(self, assignment: Assignment) -> None: if plan.assignment_status != expected_status: raise RuntimeError("server returned an inconsistent assignment status") - definitions = {(item.eval_key, item.eval_version): item for item in selected} + definitions = { + (item.eval_key, item.eval_version): (item, local) + for item, local in selected + } run_definitions: list[tuple[str, EvalDefinition]] = [] run_ids: set[str] = set() for run in plan.runs: if run.evaluation_run_id in run_ids: raise RuntimeError("server returned a duplicate evaluation run id") run_ids.add(run.evaluation_run_id) - definition = definitions.pop((run.eval_key, run.eval_version), None) - if definition is None: + selected_definition = definitions.pop( + (run.eval_key, run.eval_version), None + ) + if selected_definition is None: raise RuntimeError("server returned an unrequested evaluation run") + descriptor, local = selected_definition + if run.execution_mode is not descriptor.execution_mode: + raise RuntimeError("server changed the evaluation execution mode") + if run.execution_mode is ExecutionMode.LOCAL: + if local is None: + raise RuntimeError("local evaluation definition is unavailable") + definition = local + else: + if not run.evaluator_source or not run.source_checksum: + raise RuntimeError("server omitted managed evaluation source") + expected = source_checksum( + descriptor.condition_source, run.evaluator_source + ) + if expected != run.source_checksum or ( + descriptor.source_checksum + and descriptor.source_checksum != run.source_checksum + ): + raise RuntimeError("managed evaluation source checksum mismatch") + definition = EvalDefinition( + eval_key=descriptor.eval_key, + display_name=descriptor.display_name, + eval_version=descriptor.eval_version, + result_kind=descriptor.result_kind, + labels=descriptor.labels, + function=compile_evaluator(run.evaluator_source), + condition=None, + on_cancel=None, + timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, + ) run_definitions.append((run.evaluation_run_id, definition)) if definitions and not plan.idempotent_replay: raise RuntimeError("server omitted a selected evaluation run") @@ -479,12 +569,21 @@ async def _heartbeat( }, ) self._increment("heartbeat_failures") + except Exception as error: # noqa: BLE001 - keep lease renewal alive + logger.warning( + "evaluator heartbeat error", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + self._increment("heartbeat_failures") - @staticmethod - async def _invoke(function, session): + async def _invoke(self, function, session): if inspect.iscoroutinefunction(function): return await function(session) - result = await asyncio.to_thread(function, session) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(self._eval_executor, function, session) if inspect.isawaitable(result): return await result return result @@ -493,6 +592,35 @@ async def _invoke(function, session): def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: return SkippedEval(definition.eval_key, definition.eval_version, reason) + @staticmethod + def _skipped_descriptor( + definition: AssignmentDefinition, reason: str + ) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + async def _assignment_definitions( + self, assignment: Assignment + ) -> tuple[AssignmentDefinition, ...]: + if assignment.definitions_url: + response = await self._call_client( + self.client.definitions, + assignment, + worker_id=self.config.worker_id, + ) + if response.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned definitions for another assignment") + return response.definitions + return tuple( + AssignmentDefinition( + eval_key=item.eval_key, + display_name=item.display_name, + eval_version=item.eval_version, + result_kind=item.result_kind, + labels=item.labels, + ) + for item in self.evaluator.definitions + ) + def _reap_finished(self) -> None: done = {task for task in self._active if task.done()} self._active.difference_update(done) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py new file mode 100644 index 00000000..b3c7da9f --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -0,0 +1,170 @@ +"""Restricted deterministic expression compiler for server-authored evaluations. + +The managed worker never executes a module, statements, imports, or ambient +builtins from tenant-authored source. Definitions are single Python expressions +evaluated with a small constructor/helper surface and the immutable transcript +bound as ``session``. +""" + +from __future__ import annotations + +import ast +import hashlib +from collections.abc import Callable +from typing import Any + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalResult, + Metric, + Score, +) + +MAX_CONDITION_SOURCE_BYTES = 16 * 1024 +MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024 + +_ALLOWED_NODES = ( + ast.Expression, + ast.BoolOp, + ast.BinOp, + ast.UnaryOp, + ast.IfExp, + ast.Dict, + ast.Set, + ast.List, + ast.Tuple, + ast.ListComp, + ast.SetComp, + ast.DictComp, + ast.GeneratorExp, + ast.comprehension, + ast.Compare, + ast.Call, + ast.FormattedValue, + ast.JoinedStr, + ast.Constant, + ast.Name, + ast.Load, + ast.Store, + ast.Attribute, + ast.Subscript, + ast.Slice, + ast.keyword, + ast.And, + ast.Or, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.USub, + ast.UAdd, + ast.Not, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, +) + +_SAFE_GLOBALS = { + "__builtins__": {}, + "Assertion": Assertion, + "ConditionResult": ConditionResult, + "EvalResult": EvalResult, + "Metric": Metric, + "Score": Score, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + "enumerate": enumerate, + "float": float, + "int": int, + "len": len, + "list": list, + "max": max, + "min": min, + "range": range, + "round": round, + "set": set, + "sorted": sorted, + "str": str, + "sum": sum, + "tuple": tuple, +} + + +class UnsafeEvaluatorSource(ValueError): + """Raised before any disallowed server-authored source can execute.""" + + +def source_checksum(condition_source: str | None, evaluator_source: str) -> str: + payload = (condition_source or "").encode("utf-8") + b"\0" + evaluator_source.encode( + "utf-8" + ) + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _compile(source: str, *, field_name: str, maximum: int) -> Any: + if not isinstance(source, str) or not source.strip(): + raise UnsafeEvaluatorSource(f"{field_name} must not be empty") + if len(source.encode("utf-8")) > maximum: + raise UnsafeEvaluatorSource(f"{field_name} exceeds {maximum} bytes") + try: + tree = ast.parse(source, mode="eval") + except SyntaxError as error: + raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + for node in ast.walk(tree): + if not isinstance(node, _ALLOWED_NODES): + raise UnsafeEvaluatorSource( + f"{field_name} contains disallowed syntax: {type(node).__name__}" + ) + if isinstance(node, ast.Attribute) and node.attr.startswith("_"): + raise UnsafeEvaluatorSource( + f"{field_name} may not access private or dunder attributes" + ) + if isinstance(node, ast.Name) and node.id.startswith("_"): + raise UnsafeEvaluatorSource(f"{field_name} may not access private names") + return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) + + +def compile_condition(source: str) -> Callable[[Any], bool | ConditionResult]: + code = _compile( + source, + field_name="condition_source", + maximum=MAX_CONDITION_SOURCE_BYTES, + ) + + def condition(session: Any) -> bool | ConditionResult: + value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + if not isinstance(value, (bool, ConditionResult)): + raise TypeError("condition_source must return bool or ConditionResult") + return value + + return condition + + +def compile_evaluator(source: str) -> Callable[[Any], EvalResult]: + code = _compile( + source, + field_name="evaluator_source", + maximum=MAX_EVALUATOR_SOURCE_BYTES, + ) + + def evaluate(session: Any) -> EvalResult: + value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + if not isinstance(value, EvalResult): + raise TypeError("evaluator_source must return EvalResult") + return value + + return evaluate diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index fdc1a0cf..5998f535 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -1,5 +1,5 @@ { - "fixture_revision": "evaluator-v2-2026-08-28.2", + "fixture_revision": "evaluator-v2-2026-08-28.3", "protocol": { "supported_major_versions": ["2"], "transcript_schema_version": "2", @@ -9,6 +9,7 @@ "register": "/v1/evaluator/workers/register", "claim": "/v1/evaluator/assignments/claim", "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "definitions": "/v1/evaluator/assignments/{assignment_id}/definitions", "plan": "/v1/evaluator/assignments/{assignment_id}/plan", "heartbeat": "/v1/evaluator/runs/heartbeat", "result": "/v1/evaluator/runs/{evaluation_run_id}/result", @@ -102,7 +103,26 @@ "environment": "production", "trigger_reason": "agent_end", "event_count": 42, - "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript" + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript", + "definitions_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/definitions" + } + ] + }, + "definitions_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"], + "execution_mode": "python", + "condition_source": null, + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 } ] }, @@ -155,7 +175,11 @@ { "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", "eval_key": "tool_efficiency", - "eval_version": "1.2.0" + "eval_version": "1.2.0", + "execution_mode": "python", + "evaluator_source": "EvalResult(score=Score(1.0))", + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 } ] }, diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py index 7513e306..d74c60dc 100644 --- a/sdk/python/tests/test_evaluator_client.py +++ b/sdk/python/tests/test_evaluator_client.py @@ -163,6 +163,28 @@ def opener(request, timeout): assert captured.get_header("X-failproofai-lease-generation") == "3" +def test_definitions_use_the_server_supplied_path_and_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["definitions_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + response = client.definitions(assignment, worker_id="worker-7") + + assert response.assignment_id == assignment.assignment_id + assert response.definitions[0].execution_mode.value == "python" + assert captured.full_url.endswith(assignment.definitions_url) + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + def test_constructor_rejects_unsafe_or_incomplete_configuration(): with pytest.raises(ValueError, match="absolute"): EvaluatorClient(base_url="localhost:8080", credential="secret") @@ -172,6 +194,16 @@ def test_constructor_rejects_unsafe_or_incomplete_configuration(): EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") +def test_private_cluster_http_requires_an_explicit_opt_in(): + with pytest.raises(ValueError, match="must use https"): + EvaluatorClient(base_url="http://server:8080", credential="secret") + EvaluatorClient( + base_url="http://server:8080", + credential="secret", + allow_insecure_http=True, + ) + + def test_protocol_redirect_does_not_forward_the_bearer_credential(): exfiltration_attempts = [] diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py index a7fd18ee..e9c2cc89 100644 --- a/sdk/python/tests/test_evaluator_protocol.py +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -9,6 +9,7 @@ from failproofai_sdk.evaluator import ( ClaimRequest, ClaimResponse, + DefinitionsResponse, ErrorResponse, HeartbeatRequest, HeartbeatResponse, @@ -38,6 +39,7 @@ def _contract(): ("register_response", RegisterResponse), ("claim_request", ClaimRequest), ("claim_response", ClaimResponse), + ("definitions_response", DefinitionsResponse), ("transcript_response", SessionTranscript), ("plan_request", PlanRequest), ("plan_response", PlanResponse), @@ -181,8 +183,9 @@ def test_fixture_constants_match_the_sdk_contract(): } assert contract["http"] == { "register": protocol.REGISTER_PATH, - "claim": protocol.CLAIM_PATH, - "transcript": protocol.TRANSCRIPT_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "definitions": protocol.DEFINITIONS_PATH, "plan": protocol.PLAN_PATH, "heartbeat": protocol.HEARTBEAT_PATH, "result": protocol.RESULT_PATH, diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index bc770e5c..d7893bd5 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -2,6 +2,7 @@ import asyncio import json +import threading import time from dataclasses import replace from pathlib import Path @@ -9,11 +10,14 @@ import pytest from failproofai_sdk.evaluator import ( + AssignmentDefinition, ClaimResponse, ConditionResult, + DefinitionsResponse, EvalResult, Evaluator, EvaluatorAPIError, + ExecutionMode, HeartbeatResponse, PlannedRun, PlanResponse, @@ -23,6 +27,7 @@ SessionTranscript, WorkerConfig, WorkerRuntime, + source_checksum, ) FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" @@ -38,6 +43,7 @@ def __init__(self): self.assignment = ClaimResponse.from_wire( samples["claim_response"] ).assignments[0] + self.assignment = replace(self.assignment, definitions_url="") self.session = SessionTranscript.from_wire(samples["transcript_response"]) self.register_requests = [] self.claim_requests = [] @@ -94,6 +100,131 @@ def _runtime(evaluator, client): ) +def test_managed_definition_is_fetched_verified_and_executed(): + source = "EvalResult(score=Score(0.75, passed=True), summary='hosted')" + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, source), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=source, + source_checksum=source_checksum(None, source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "succeeded" + assert result.summary == "hosted" + assert result.results[0].numeric_value == 0.75 + + +def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): + evaluator = Evaluator(name="parallel", version="1") + lock = threading.Lock() + active = 0 + peak = 0 + + def measured(_session): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.04) + with lock: + active -= 1 + return EvalResult(score=Score(1)) + + for index in range(5): + evaluator.eval( + f"eval_{index}", + version="1", + when=lambda session, index=index: ( + index < 3 if session.session_id == "session-a" else index >= 3 + ), + )(measured) + + class ParallelClient(FakeClient): + def transcript(self, assignment, *, worker_id): + assert worker_id == "worker-test" + return replace( + self.session, + assignment_id=assignment.assignment_id, + session_id=assignment.session_id, + session_revision_id=assignment.session_revision_id, + ) + + client = ParallelClient() + first = replace( + client.assignment, + assignment_id="assignment-a", + session_id="session-a", + session_revision_id="revision-a", + ) + second = replace( + client.assignment, + assignment_id="assignment-b", + session_id="session-b", + session_revision_id="revision-b", + ) + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + await asyncio.gather( + runtime.process_assignment(first), runtime.process_assignment(second) + ) + + asyncio.run(exercise()) + + assert peak == 2 + assert len(client.submissions) == 5 + assert client.heartbeats + + def test_condition_failures_are_isolated_and_plan_is_declared_first(): evaluator = Evaluator(name="test", version="1") @@ -712,3 +843,5 @@ async def measured(session): ) asyncio.run(runtime.process_assignment(client.assignment)) assert peak == 1 + DefinitionsResponse, + ExecutionMode, diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py new file mode 100644 index 00000000..91e5a7f2 --- /dev/null +++ b/sdk/python/tests/test_evaluator_source.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import pytest + +from failproofai_sdk.evaluator import EvalResult, Score +from failproofai_sdk.evaluator.source import ( + MAX_EVALUATOR_SOURCE_BYTES, + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + + +class Session: + event_count = 3 + + +def test_restricted_expressions_can_evaluate_conditions_and_results(): + assert compile_condition("session.event_count > 0")(Session()) is True + result = compile_evaluator("EvalResult(score=Score(0.75, passed=True))")( + Session() + ) + assert isinstance(result, EvalResult) + assert result.score == Score(0.75, passed=True) + + +@pytest.mark.parametrize( + "source", + [ + "__import__('os').system('id')", + "session.__class__", + "(lambda: 1)()", + "[x for x in ().__class__.__base__.__subclasses__()]", + ], +) +def test_restricted_expressions_reject_escape_primitives(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +def test_restricted_expressions_reject_statements_and_oversized_source(): + with pytest.raises(UnsafeEvaluatorSource, match="one expression"): + compile_evaluator("import os") + with pytest.raises(UnsafeEvaluatorSource, match="exceeds"): + compile_evaluator("x" * (MAX_EVALUATOR_SOURCE_BYTES + 1)) + + +def test_result_and_condition_types_are_checked_at_runtime(): + with pytest.raises(TypeError, match="EvalResult"): + compile_evaluator("True")(Session()) + with pytest.raises(TypeError, match="bool or ConditionResult"): + compile_condition("1")(Session()) + + +def test_source_checksum_covers_condition_and_evaluator_together(): + base = source_checksum(None, "EvalResult()") + assert base == source_checksum(None, "EvalResult()") + assert base != source_checksum("True", "EvalResult()") + assert base != source_checksum(None, "EvalResult(summary='changed')") From e9701df73afa4982eef6522af996017551af9375 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Sun, 30 Aug 2026 23:57:17 +0530 Subject: [PATCH 05/20] fix(evaluator): harden managed source sandbox and contain poison definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial review of the server-authored `execution_mode='python'` evaluations (which run in the shared managed pod) found the AST sandbox escapable several ways: `str.format`/`format_map` C-level field traversal, generator/frame introspection (`gi_frame.f_globals`) that reached the eval globals and could poison a process-shared namespace across evaluations, and `type.mro()` type-object reach — none of which start with `_`, so the dunder guard never saw them. Replace the attribute denylist with a **default-deny allowlist** (the transcript data surface plus pure string/collection methods), give each eval **fresh per-call globals** so nothing persists between evaluations, and reject any result whose text embeds a runtime object repr (`<... at 0x...>`, the heap-pointer/ASLR disclosure that falls out of any bound method's repr) at the output boundary. Drop `enumerate` and bare generator expressions — both were gratuitous pointer-repr sources. Also compile managed source lazily inside the per-run executor, so a definition the sandbox rejects dead-letters as one bounded `failed`/`eval_error` run instead of crashing the assignment and being reclaimed until its attempt budget is spent. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 17 ++ .../failproofai_sdk/evaluator/runtime.py | 20 ++- .../failproofai_sdk/evaluator/source.py | 150 ++++++++++++++++-- sdk/python/tests/test_evaluator_runtime.py | 70 ++++++++ sdk/python/tests/test_evaluator_source.py | 125 +++++++++++++++ 5 files changed, 371 insertions(+), 11 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 74d6b5b3..4a6f5143 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -19,6 +19,23 @@ it ships. - Retire the old inbound evaluator boundary and add evaluator authoring plus the outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator` namespace. +- Harden the managed-evaluator source sandbox against a class of escapes an + adversarial review found: `str.format`/`format_map` C-level field traversal, + generator/frame introspection (`gi_frame.f_globals`) that reached the eval + globals and could poison a process-shared namespace across evaluations, and + `type.mro()` type-object reach. Attribute access is now **default-deny** (an + allowlist of the transcript data surface plus pure string/collection methods, + so every current and future introspection attribute is rejected), each eval + runs with **fresh per-call globals**, and a result whose text embeds a runtime + object repr (`<... at 0x...>`, a heap-pointer/ASLR disclosure that falls out of + any bound method's repr) is rejected at the output boundary. `enumerate` and + bare generator expressions are no longer permitted — both were gratuitous + pointer-repr sources; use `range(len(...))` and list/set/dict comprehensions. +- Contain a poison managed definition to its own run: source is now compiled + lazily inside the per-run executor, so a definition the sandbox rejects + dead-letters as one bounded `failed`/`eval_error` run instead of crashing the + assignment task and forcing it to be reclaimed until its attempt budget runs + out. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index f96d205f..f39dbb14 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -57,6 +57,24 @@ def _utc_now() -> str: ) +def _deferred_managed_eval(source: str): + """Compile server-authored source lazily, at invocation time. + + Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``). + Building the definition with this thunk instead of a pre-compiled function + routes that failure through the same per-run ``try/except`` that turns any + evaluation error into a bounded ``FAILED`` result — so a poison definition + dead-letters cleanly as one failed run instead of raising out of assignment + setup, crashing the task, and forcing the whole assignment to be reclaimed + and retried until its attempt budget is exhausted. + """ + + def evaluate(session: Any) -> Any: + return compile_evaluator(source)(session) + + return evaluate + + def _positive_int(name: str, default: int) -> int: raw = os.environ.get(name) if raw is None: @@ -418,7 +436,7 @@ async def process_assignment(self, assignment: Assignment) -> None: eval_version=descriptor.eval_version, result_kind=descriptor.result_kind, labels=descriptor.labels, - function=compile_evaluator(run.evaluator_source), + function=_deferred_managed_eval(run.evaluator_source), condition=None, on_cancel=None, timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index b3c7da9f..26eb4b3a 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -10,6 +10,7 @@ import ast import hashlib +import re from collections.abc import Callable from typing import Any @@ -37,7 +38,11 @@ ast.ListComp, ast.SetComp, ast.DictComp, - ast.GeneratorExp, + # `ast.GeneratorExp` is intentionally NOT allowed: a bare generator object's + # default repr is ``, which leaks a live host + # heap address (an ASLR/memory-layout disclosure) the moment it is coerced to + # a string into any result field. List/set/dict comprehensions render as their + # data (`[...]`, `{...}`) and cover the same ground — wrap a generator in `[]`. ast.comprehension, ast.Compare, ast.Call, @@ -87,7 +92,9 @@ "any": any, "bool": bool, "dict": dict, - "enumerate": enumerate, + # `enumerate` is intentionally excluded: an enumerate object's default repr is + # ``, leaking a live host heap address into any + # result field. Index-aware iteration can use `range(len(...))` instead. "float": float, "int": int, "len": len, @@ -104,6 +111,104 @@ } +# Attribute access is DEFAULT-DENY. A denylist is unwinnable here: dunder access +# is only one door. `str.format`/`format_map` traverse a format string's fields +# at the C level; `(x for x in [1]).gi_frame.f_globals` reaches the eval globals +# through generator/frame introspection; `str.mro()[-1]` reaches the `object` +# type — and NONE of `format`, `gi_frame`, `f_globals`, `co_names`, `mro`, ... +# start with an underscore, so the dunder guard never sees them. Rather than +# chase each introspection family, we allow ONLY the attribute names a real +# session evaluation needs: the transcript/event data surface plus a fixed set +# of pure string/collection data methods. Anything else — every current and +# future introspection attribute — is rejected. `format`/`format_map` are simply +# absent from this set, so the C-level format escape is closed too. +_ALLOWED_ATTRS = frozenset( + { + # SessionTranscript + TranscriptEvent data surface (see protocol.py). + "events", + "events_of_type", + "count", + "event_count", + "event_type", + "payload", + "id", + "ts", + "agent_id", + "environment", + "session_id", + "session_revision_id", + "assignment_id", + "started_at", + "ended_at", + "schema_version", + # dict data methods. + "get", + "keys", + "values", + "items", + # str / bytes pure data methods. + "lower", + "upper", + "strip", + "lstrip", + "rstrip", + "split", + "rsplit", + "splitlines", + "startswith", + "endswith", + "replace", + "find", + "rfind", + "index", + "join", + "title", + "capitalize", + "casefold", + "swapcase", + "isdigit", + "isalpha", + "isalnum", + "isspace", + "isnumeric", + "isdecimal", + "islower", + "isupper", + "istitle", + "zfill", + "ljust", + "rjust", + "center", + "partition", + "rpartition", + "removeprefix", + "removesuffix", + "encode", + "decode", + "hex", + # set data methods. + "union", + "intersection", + "difference", + "symmetric_difference", + "issubset", + "issuperset", + "isdisjoint", + } +) + + +def _fresh_globals() -> dict[str, Any]: + """A throwaway globals mapping for one eval call. + + Every evaluation gets its own copy — with a fresh empty ``__builtins__`` — + so that even if a future reach exposes the eval's globals (e.g. through a + frame object), a mutation cannot persist into another evaluation and poison + a shared, process-wide namespace. + """ + return {**_SAFE_GLOBALS, "__builtins__": {}} + + class UnsafeEvaluatorSource(ValueError): """Raised before any disallowed server-authored source can execute.""" @@ -129,15 +234,40 @@ def _compile(source: str, *, field_name: str, maximum: int) -> Any: raise UnsafeEvaluatorSource( f"{field_name} contains disallowed syntax: {type(node).__name__}" ) - if isinstance(node, ast.Attribute) and node.attr.startswith("_"): - raise UnsafeEvaluatorSource( - f"{field_name} may not access private or dunder attributes" - ) + if isinstance(node, ast.Attribute): + if node.attr.startswith("_"): + raise UnsafeEvaluatorSource( + f"{field_name} may not access private or dunder attributes" + ) + if node.attr not in _ALLOWED_ATTRS: + raise UnsafeEvaluatorSource( + f"{field_name} may not access attribute '{node.attr}'" + ) if isinstance(node, ast.Name) and node.id.startswith("_"): raise UnsafeEvaluatorSource(f"{field_name} may not access private names") return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) +# CPython's default object repr — `<... at 0x7f...>` — embeds a live heap +# address (an ASLR/memory-layout disclosure). An expression cannot be stopped +# from producing such a repr at the source level: it falls out of `str()` on any +# bound method of an allowed object (`str(payload.get)`), and those methods must +# stay reachable. So the disclosure is closed at the OUTPUT boundary instead: a +# result whose text embeds this signature is rejected. The result types are all +# frozen dataclasses with pointer-free reprs, so scanning the value's repr sees +# every user-controlled string field. The pattern is the interpreter's own repr +# grammar, which authored reasoning/summaries never legitimately contain. +_OBJECT_REPR = re.compile(r"<[^<>]* at 0x[0-9a-fA-F]+") + + +def _forbid_object_reprs(field_name: str, value: Any) -> Any: + if _OBJECT_REPR.search(repr(value)): + raise UnsafeEvaluatorSource( + f"{field_name} result may not embed a runtime object repr" + ) + return value + + def compile_condition(source: str) -> Callable[[Any], bool | ConditionResult]: code = _compile( source, @@ -146,10 +276,10 @@ def compile_condition(source: str) -> Callable[[Any], bool | ConditionResult]: ) def condition(session: Any) -> bool | ConditionResult: - value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 if not isinstance(value, (bool, ConditionResult)): raise TypeError("condition_source must return bool or ConditionResult") - return value + return _forbid_object_reprs("condition_source", value) return condition @@ -162,9 +292,9 @@ def compile_evaluator(source: str) -> Callable[[Any], EvalResult]: ) def evaluate(session: Any) -> EvalResult: - value = eval(code, _SAFE_GLOBALS, {"session": session}) # noqa: S307 + value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 if not isinstance(value, EvalResult): raise TypeError("evaluator_source must return EvalResult") - return value + return _forbid_object_reprs("evaluator_source", value) return evaluate diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index d7893bd5..17d01d3e 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -162,6 +162,76 @@ def plan(self, assignment_id, request): assert result.results[0].numeric_value == 0.75 +def test_managed_definition_that_fails_to_compile_dead_letters_as_one_failed_run(): + # Unsafe/malformed server-authored source is rejected by the sandbox at + # compile time. That rejection must surface as a single bounded FAILED run, + # NOT as an exception out of assignment setup that crashes the task and + # forces the whole assignment to be reclaimed and retried. + unsafe = ( + 'EvalResult(score=Score(1.0), ' + 'reasoning="{0.__class__}".format(session))' + ) + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, unsafe), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=unsafe, + source_checksum=source_checksum(None, unsafe), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + # Must NOT raise — the poison definition is contained to its own run. + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "failed" + assert result.error_code == "eval_error" + # Nothing derived from the rejected source may be reported. + assert result.results == () + assert result.summary is None + + def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): evaluator = Evaluator(name="parallel", version="1") lock = threading.Lock() diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 91e5a7f2..c583319d 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -39,6 +39,131 @@ def test_restricted_expressions_reject_escape_primitives(source): compile_evaluator(source) +@pytest.mark.parametrize( + "source", + [ + # `str.format` / `str.format_map` traverse a format string's field spec + # at the C level, reaching attributes the AST dunder guard never sees. + # These reached real `__builtins__` before the denylist landed. + '"{0.__class__.__init__.__globals__[__builtins__][__import__]}".format(session)', + '"{0.__class__}".format(session)', + 'str.format("{0.__class__}", session)', + '"{a.__class__}".format_map({"a": session})', + # A reasoning string is where a leak would surface — block it there too. + 'EvalResult(score=Score(0.5), reasoning="{0.__class__}".format(session))', + ], +) +def test_restricted_expressions_reject_format_string_traversal(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +@pytest.mark.parametrize( + "source", + [ + # Generator/frame/code introspection reaches the eval globals and, via + # dict.update on them, could poison a shared namespace. None of these + # attribute names start with "_", so only the default-deny allowlist + # stops them. + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals))", + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_code))", + 'EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals.update({"P": 1})))', + # `mro` is a public method on the type metaclass; it reaches `object`. + "EvalResult(score=Score(0.5), reasoning=str(str.mro()[-1]))", + "EvalResult(score=Score(0.5), reasoning=str(int.mro()))", + # A live function's identity would leak a heap pointer (ASLR defeat). + "EvalResult(score=Score(0.5), reasoning=str(EvalResult.result_items))", + ], +) +def test_restricted_expressions_reject_introspection_attributes(source): + with pytest.raises(UnsafeEvaluatorSource, match="attribute"): + compile_evaluator(source) + + +def test_no_reachable_construct_leaks_a_heap_pointer_repr(): + # An object's default repr (`<... object at 0x...>`) leaks a live host heap + # address (ASLR/memory-layout disclosure) if coerced into a result field. + # Generator expressions are rejected at compile; `enumerate` is not bound, so + # it raises NameError at run time and becomes a bounded failed run instead of + # a disclosure. Either way, a pointer must never reach a result string. + import re + + from failproofai_sdk.evaluator.source import _SAFE_GLOBALS + + with pytest.raises(UnsafeEvaluatorSource, match="GeneratorExp"): + compile_evaluator("EvalResult(score=Score(1.0), reasoning=str((x for x in [1])))") + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(enumerate([1])))" + ) + with pytest.raises(NameError): + evaluate(Session()) + + # No value bound into the evaluation namespace reprs to a heap pointer. + pointer = re.compile(r"0x[0-9a-fA-F]+") + leaky = { + name: repr(value) + for name, value in _SAFE_GLOBALS.items() + if name != "__builtins__" and pointer.search(repr(value)) + } + assert leaky == {} + + +@pytest.mark.parametrize( + "inner", + [ + # A bound method's repr leaks the underlying object's heap pointer, and + # these methods are allowlisted (needed by real evals) so they cannot be + # removed. The OUTPUT guard rejects the disclosure wherever it rides out. + "session.events[0].payload.get", + "''.join", + "'x'.encode", + "'a,b'.split", + ], +) +def test_object_repr_pointer_disclosure_is_rejected_at_the_output(inner): + class Sess: + class _E: + payload = {"k": "v"} + + events = (_E(),) + + for field in ( + f'EvalResult(score=Score(1.0), reasoning=str({inner}))', + f'EvalResult(score=Score(1.0), summary=str({inner}))', + f'EvalResult(score=Score(1.0, display_value=str({inner})))', + f'EvalResult(score=Score(1.0), labels=(str({inner}),))', + ): + evaluate = compile_evaluator(field) + with pytest.raises(UnsafeEvaluatorSource, match="object repr"): + evaluate(Sess()) + + +def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next(): + # Even setting aside the allowlist, one evaluation must not be able to leave + # state behind for the next. Compiling and running twice must not share a + # mutable namespace. + from failproofai_sdk.evaluator.source import _fresh_globals + + first = _fresh_globals() + second = _fresh_globals() + assert first is not second + assert first["__builtins__"] is not second["__builtins__"] + first["__poison__"] = "leaked" + assert "__poison__" not in second + + +def test_format_denylist_does_not_block_legitimate_string_methods(): + # The fix is a targeted denylist of `format`/`format_map`, not a ban on all + # string methods — ordinary evaluations must still compile and run. + evaluate = compile_evaluator( + 'EvalResult(score=Score(0.9, passed=True), ' + 'reasoning="tools=" + str(session.event_count).upper())' + ) + result = evaluate(Session()) + assert result.reasoning == "tools=3" + + def test_restricted_expressions_reject_statements_and_oversized_source(): with pytest.raises(UnsafeEvaluatorSource, match="one expression"): compile_evaluator("import os") From b6f5eb84d2327535446c64443716f585ca3595fe Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 14:56:33 +0530 Subject: [PATCH 06/20] feat(evaluator): normal short polling instead of long-poll in the worker The v2 worker long-polled the claim endpoint (wait_seconds, server held the request open up to 25s), which ties up a server request handler per idle worker and does not match the normal-polling cadence of our other cloud surfaces. The worker now polls normally: claim returns immediately, and on an empty claim the worker sleeps the server-advertised poll_interval_seconds (from the register response, default 10s) before polling again. Wire changes (mirrored with the server): ClaimRequest drops wait_seconds and MAX_CLAIM_WAIT_SECONDS is removed; RegisterResponse gains poll_interval_seconds, which the worker adopts like heartbeat_interval_seconds and rejects if non-positive. WorkerConfig drops claim_wait_seconds and the request_timeout_seconds > claim_wait_seconds constraint. Contract fixture updated in lockstep with the agenteye copy. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 8 +++ .../failproofai_sdk/evaluator/protocol.py | 9 ++- .../failproofai_sdk/evaluator/runtime.py | 27 +++----- .../tests/fixtures/evaluator_v2/contract.json | 6 +- sdk/python/tests/test_evaluator_http_e2e.py | 3 +- sdk/python/tests/test_evaluator_protocol.py | 2 +- sdk/python/tests/test_evaluator_runtime.py | 64 ++++++++++++++++--- 7 files changed, 85 insertions(+), 34 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 4a6f5143..a1071320 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -36,6 +36,14 @@ it ships. dead-letters as one bounded `failed`/`eval_error` run instead of crashing the assignment task and forcing it to be reclaimed until its attempt budget runs out. +- Switch the worker from long-polling to **normal (short) polling**, matching the + cadence of our other cloud surfaces. `claim` no longer sends `wait_seconds` and + the server returns immediately; when a claim comes back empty the worker sleeps + the server-advertised `poll_interval_seconds` (from the register response, + default 10 s) before polling again, instead of holding a request open for up to + 25 s. Removes the `claim_wait_seconds` config knob and the + `request_timeout_seconds > claim_wait_seconds` constraint; the poll cadence is + now tuned centrally by the server, not per worker. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index e8fa156c..468bd5e4 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -24,7 +24,10 @@ HEARTBEAT_INTERVAL_SECONDS = 30 LEASE_DURATION_SECONDS = 120 -MAX_CLAIM_WAIT_SECONDS = 25 +# Fallback poll cadence if the register response omits poll_interval_seconds. The +# worker prefers the server-advertised value; claims are normal short polls, never +# long-polls, so this only bounds idle latency, not connection lifetime. +DEFAULT_POLL_INTERVAL_SECONDS = 10 MAX_ATTEMPTS = 5 MAX_CATALOG_DEFINITIONS = 100 @@ -245,6 +248,7 @@ class RegisterResponse(WireModel): evaluator_kind: EvaluatorKind heartbeat_interval_seconds: int lease_duration_seconds: int + poll_interval_seconds: int claim_limit: int disabled_definitions: tuple[str, ...] = () protocol_version: str = PROTOCOL_VERSION @@ -257,6 +261,7 @@ def from_wire(cls, data: Mapping[str, Any]) -> RegisterResponse: evaluator_kind=_enum(EvaluatorKind, data, "evaluator_kind"), heartbeat_interval_seconds=_integer(data, "heartbeat_interval_seconds"), lease_duration_seconds=_integer(data, "lease_duration_seconds"), + poll_interval_seconds=_integer(data, "poll_interval_seconds"), claim_limit=_integer(data, "claim_limit"), disabled_definitions=_string_list(data, "disabled_definitions"), ) @@ -267,7 +272,6 @@ class ClaimRequest(WireModel): worker_id: str catalog_revision: str capacity: int - wait_seconds: int protocol_version: str = PROTOCOL_VERSION @classmethod @@ -277,7 +281,6 @@ def from_wire(cls, data: Mapping[str, Any]) -> ClaimRequest: worker_id=_string(data, "worker_id"), catalog_revision=_string(data, "catalog_revision"), capacity=_integer(data, "capacity"), - wait_seconds=_integer(data, "wait_seconds"), ) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index f39dbb14..7be4e7a1 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -24,8 +24,8 @@ ) from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient from failproofai_sdk.evaluator.protocol import ( + DEFAULT_POLL_INTERVAL_SECONDS, MAX_CLAIM_CAPACITY, - MAX_CLAIM_WAIT_SECONDS, MAX_WORKER_ID_BYTES, Assignment, AssignmentDefinition, @@ -106,7 +106,6 @@ class WorkerConfig: credential: str worker_id: str max_concurrency: int = 1 - claim_wait_seconds: int = 20 request_timeout_seconds: int = 30 drain_timeout_seconds: int = 60 allow_insecure_http: bool = False @@ -135,9 +134,6 @@ def from_env(cls) -> WorkerConfig: credential=credential, worker_id=worker_id, max_concurrency=_positive_int("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), - claim_wait_seconds=_positive_int( - "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", 20 - ), request_timeout_seconds=_positive_int( "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30 ), @@ -152,16 +148,6 @@ def from_env(cls) -> WorkerConfig: raise ValueError( f"FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds {MAX_CLAIM_CAPACITY}" ) - if config.claim_wait_seconds > MAX_CLAIM_WAIT_SECONDS: - raise ValueError( - "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS exceeds " - f"{MAX_CLAIM_WAIT_SECONDS}" - ) - if config.request_timeout_seconds <= config.claim_wait_seconds: - raise ValueError( - "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS must exceed " - "FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS" - ) return config @@ -184,6 +170,7 @@ def __init__( self._stopping = asyncio.Event() self._active: set[asyncio.Task[None]] = set() self._heartbeat_interval = 30 + self._poll_interval = DEFAULT_POLL_INTERVAL_SECONDS self._claim_limit = config.max_concurrency self._lease_duration = 120 self._disabled_definitions: set[str] = set() @@ -213,10 +200,12 @@ async def register(self) -> None: self._increment("registration_failure") raise self._heartbeat_interval = response.heartbeat_interval_seconds + self._poll_interval = response.poll_interval_seconds self._lease_duration = response.lease_duration_seconds self._claim_limit = min(self.config.max_concurrency, response.claim_limit) if ( self._heartbeat_interval <= 0 + or self._poll_interval <= 0 or self._lease_duration <= self._heartbeat_interval or self._claim_limit <= 0 ): @@ -245,7 +234,6 @@ async def run_forever(self) -> None: worker_id=self.config.worker_id, catalog_revision=self.evaluator.catalog_revision, capacity=capacity, - wait_seconds=self.config.claim_wait_seconds, ), ) except EvaluatorAPIError as error: @@ -272,6 +260,12 @@ async def run_forever(self) -> None: task = asyncio.create_task(self.process_assignment(assignment)) self._active.add(task) self._increment("assignments_claimed", len(assignments)) + if not assignments: + # Normal short poll: the server returns immediately, so when + # nothing is queued we wait the advertised interval before + # polling again instead of hot-looping. When work IS returned + # we loop straight back to drain any backlog up to capacity. + await self._wait_or_stop(float(self._poll_interval)) finally: await self.drain() self._eval_executor.shutdown(wait=False, cancel_futures=True) @@ -284,7 +278,6 @@ async def run_once(self) -> int: worker_id=self.config.worker_id, catalog_revision=self.evaluator.catalog_revision, capacity=self._claim_limit, - wait_seconds=self.config.claim_wait_seconds, ), ) assignments = self._validated_assignments( diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index 5998f535..afd09a07 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -19,7 +19,7 @@ "timing": { "heartbeat_interval_seconds": 30, "lease_duration_seconds": 120, - "max_claim_wait_seconds": 25, + "poll_interval_seconds": 10, "max_attempts": 5 }, "limits": { @@ -80,6 +80,7 @@ "evaluator_kind": "customer", "heartbeat_interval_seconds": 30, "lease_duration_seconds": 120, + "poll_interval_seconds": 10, "claim_limit": 4, "disabled_definitions": [] }, @@ -87,8 +88,7 @@ "protocol_version": "2", "worker_id": "pod-7f8c9", "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", - "capacity": 2, - "wait_seconds": 20 + "capacity": 2 }, "claim_response": { "protocol_version": "2", diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py index 475e0bf3..508d92ba 100644 --- a/sdk/python/tests/test_evaluator_http_e2e.py +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -111,6 +111,7 @@ def do_POST(self) -> None: "evaluator_kind": kind, "heartbeat_interval_seconds": 30, "lease_duration_seconds": 120, + "poll_interval_seconds": 10, "claim_limit": body["max_concurrency"], "disabled_definitions": [], }, @@ -410,7 +411,6 @@ def _claim(client: EvaluatorClient, worker_id: str): worker_id=worker_id, catalog_revision="sha256:" + "a" * 64, capacity=1, - wait_seconds=0, ) ) @@ -432,7 +432,6 @@ def completion_present(session): server_url=state.base_url, credential="customer-a-token", worker_id="worker-a", - claim_wait_seconds=1, ), client=_client(state, "customer-a-token"), ) diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py index e9c2cc89..99a874b6 100644 --- a/sdk/python/tests/test_evaluator_protocol.py +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -195,7 +195,7 @@ def test_fixture_constants_match_the_sdk_contract(): assert contract["timing"] == { "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS, "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS, - "max_claim_wait_seconds": protocol.MAX_CLAIM_WAIT_SECONDS, + "poll_interval_seconds": protocol.DEFAULT_POLL_INTERVAL_SECONDS, "max_attempts": protocol.MAX_ATTEMPTS, } assert contract["limits"] == { diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index 17d01d3e..8c637d9e 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -94,7 +94,6 @@ def _runtime(evaluator, client): credential="secret", worker_id="worker-test", max_concurrency=2, - claim_wait_seconds=1, ), client=client, ) @@ -666,6 +665,7 @@ def register(self, request): evaluator_kind=self._kind(), heartbeat_interval_seconds=120, lease_duration_seconds=120, + poll_interval_seconds=10, claim_limit=1, ) @@ -713,6 +713,33 @@ async def stop_after_wait(seconds): } +def test_idle_claim_waits_the_advertised_poll_interval_before_polling_again(): + # Normal short polling: an empty claim returns immediately (no long-poll), so + # the worker sleeps the server-advertised poll_interval_seconds — 10 in the + # fixture register response — instead of hot-looping. The claim request also no + # longer carries a wait_seconds field. + evaluator = Evaluator(name="test", version="1") + + class IdleClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=()) + + runtime = _runtime(evaluator, IdleClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [10.0] + assert len(runtime.client.claim_requests) == 1 + assert not hasattr(runtime.client.claim_requests[0], "wait_seconds") + + def test_nonretryable_claim_failure_stops_the_worker(): evaluator = Evaluator(name="test", version="1") @@ -776,6 +803,7 @@ def register(self, request): evaluator_kind=self._kind(), heartbeat_interval_seconds=10, lease_duration_seconds=120, + poll_interval_seconds=10, claim_limit=1, disabled_definitions=("disabled",), ) @@ -807,13 +835,33 @@ def test_worker_config_requires_dedicated_credentials(monkeypatch): WorkerConfig.from_env() -def test_worker_config_keeps_long_poll_inside_the_http_timeout(monkeypatch): - monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") - monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") - monkeypatch.setenv("FAILPROOFAI_EVALUATOR_CLAIM_WAIT_SECONDS", "20") - monkeypatch.setenv("FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", "20") - with pytest.raises(ValueError, match="must exceed"): - WorkerConfig.from_env() +def test_register_rejects_non_positive_poll_interval(): + # The worker adopts the server-advertised poll_interval_seconds (normal short + # polling — there is no long-poll wait). A non-positive interval would make the + # claim loop hot-spin, so registration must refuse it. + evaluator = Evaluator(name="test", version="1") + + class ZeroPollClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=30, + lease_duration_seconds=120, + poll_interval_seconds=0, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, ZeroPollClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False def test_worker_config_rejects_header_control_characters(monkeypatch): From 7b413466debf20c4528adf71b3abbc34162838d3 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 16:05:25 +0530 Subject: [PATCH 07/20] fix(evaluator): sandbox managed source in a killable process (hermes SEC-001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Managed (server-authored) evaluations ran in an in-process ThreadPoolExecutor with an asyncio.wait_for timeout that only cancelled the awaiter — Python cannot kill the running thread, so `sum(range(10**20))` kept burning CPU well past the timeout and tied up the sole worker slot; conditions ran with no timeout at all. Run managed conditions/evaluators in a forked child with hard RLIMIT_CPU + RLIMIT_AS + a parent-side wall-clock SIGKILL, killed on timeout before capacity is released. The kernel enforces the limits on a separate process the parent can terminate outright — the one thing a thread cannot do. Only the result crosses back, as a small pickle, with the child's exception semantics preserved. `resource` is imported at module level (never in the child) and the child does only eval->pickle->write->_exit, so the fork holds no lock another thread owns. Defense in depth at compile: reject `**` with a large/non-constant exponent and cap AST size. A managed condition the sandbox rejects now dead-letters as condition_error instead of raising out of the plan loop. Also require execution_mode on the wire (F2): a falsy/missing value was silently coerced to `local`, running a `python` definition down the customer path. Only server-authored source is isolated; customer evaluators run their own trusted code in-process. New tests cover the compute/condition bombs, the AST bounds, the fork result round-trip, and the execution_mode rejection. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 14 ++ .../failproofai_sdk/evaluator/protocol.py | 20 +- .../failproofai_sdk/evaluator/runtime.py | 46 ++-- .../failproofai_sdk/evaluator/source.py | 212 +++++++++++++++++- sdk/python/tests/test_evaluator_http_e2e.py | 8 +- sdk/python/tests/test_evaluator_protocol.py | 20 ++ sdk/python/tests/test_evaluator_source.py | 56 +++++ 7 files changed, 345 insertions(+), 31 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index a1071320..64619456 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -36,6 +36,20 @@ it ships. dead-letters as one bounded `failed`/`eval_error` run instead of crashing the assignment task and forcing it to be reclaimed until its attempt budget runs out. +- Run managed (server-authored) evaluations in a **killable forked process** with + hard `RLIMIT_CPU` + `RLIMIT_AS` + wall-clock limits, killed on timeout — so a + compute/memory bomb in a hosted definition (`sum(range(10**20))`) can no longer + exhaust the worker (SEC-001). Cancelling an in-process thread does not stop it; + a forked process the kernel bounds and the parent can `SIGKILL` does. Managed + conditions, which previously ran with no timeout at all, are sandboxed the same + way. Defense in depth at compile time: reject `**` with a large/non-constant + exponent and cap total AST size. A managed condition the sandbox rejects now + dead-letters as `condition_error` instead of raising out of the plan loop and + stranding the assignment. Only server-authored source is isolated this way; + customer evaluators still run in-process (their own trusted code). +- Require `execution_mode` on the wire instead of coercing a falsy/missing value + to `local` — a malformed value silently ran a `python` definition down the + customer path (or vice-versa); it is now a hard protocol error. - Switch the worker from long-polling to **normal (short) polling**, matching the cadence of our other cloud surfaces. `claim` no longer sends `wait_seconds` and the server returns immediately; when a claim comes back empty the worker sleeps diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index 468bd5e4..ab02c09a 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -342,11 +342,11 @@ def from_wire(cls, data: Mapping[str, Any]) -> AssignmentDefinition: eval_version=_string(data, "eval_version"), result_kind=_enum(ResultKind, data, "result_kind"), labels=_string_list(data, "labels"), - execution_mode=_enum( - ExecutionMode, - {"execution_mode": data.get("execution_mode") or "local"}, - "execution_mode", - ), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), condition_source=_optional_string(data, "condition_source"), source_checksum=_optional_string(data, "source_checksum"), timeout_seconds=timeout, @@ -524,11 +524,11 @@ def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: evaluation_run_id=_string(data, "evaluation_run_id"), eval_key=_string(data, "eval_key"), eval_version=_string(data, "eval_version"), - execution_mode=_enum( - ExecutionMode, - {"execution_mode": data.get("execution_mode") or "local"}, - "execution_mode", - ), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), evaluator_source=_optional_string(data, "evaluator_source"), source_checksum=_optional_string(data, "source_checksum"), timeout_seconds=timeout, diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 7be4e7a1..c9f89b5b 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -41,6 +41,7 @@ TerminalRunStatus, ) from failproofai_sdk.evaluator.source import ( + EvaluationTimeout, compile_condition, compile_evaluator, source_checksum, @@ -57,7 +58,7 @@ def _utc_now() -> str: ) -def _deferred_managed_eval(source: str): +def _deferred_managed_eval(source: str, timeout_seconds: int | None): """Compile server-authored source lazily, at invocation time. Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``). @@ -70,7 +71,7 @@ def _deferred_managed_eval(source: str): """ def evaluate(session: Any) -> Any: - return compile_evaluator(source)(session) + return compile_evaluator(source, timeout_seconds=timeout_seconds)(session) return evaluate @@ -331,19 +332,26 @@ async def process_assignment(self, assignment: Assignment) -> None: skipped.append(self._skipped_descriptor(descriptor, "disabled_by_server")) self._increment("conditions_skipped") continue - condition_function = ( - local.condition - if local is not None - else ( - compile_condition(descriptor.condition_source) - if descriptor.condition_source - else None - ) - ) - if condition_function is None: - selected.append((descriptor, local)) - continue try: + # Compile INSIDE the try: a managed condition the sandbox rejects + # (unsafe/malformed source) must dead-letter as `condition_error`, + # not raise out of the plan loop and strand the whole assignment + # until its retry budget is exhausted. + condition_function = ( + local.condition + if local is not None + else ( + compile_condition( + descriptor.condition_source, + timeout_seconds=descriptor.timeout_seconds, + ) + if descriptor.condition_source + else None + ) + ) + if condition_function is None: + selected.append((descriptor, local)) + continue condition = await self._invoke(condition_function, session) if isinstance(condition, ConditionResult): applicable = condition.applicable @@ -429,7 +437,10 @@ async def process_assignment(self, assignment: Assignment) -> None: eval_version=descriptor.eval_version, result_kind=descriptor.result_kind, labels=descriptor.labels, - function=_deferred_managed_eval(run.evaluator_source), + function=_deferred_managed_eval( + run.evaluator_source, + run.timeout_seconds or descriptor.timeout_seconds, + ), condition=None, on_cancel=None, timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, @@ -495,7 +506,10 @@ async def _execute_run_in_slot( summary = result.summary error_code = None error_message = None - except asyncio.TimeoutError: + except (asyncio.TimeoutError, EvaluationTimeout): + # asyncio.TimeoutError: the awaiter hit the wall-clock. EvaluationTimeout: + # the forked managed sandbox was killed by its CPU/memory/time budget — + # the real, thread-uncancellable case. Both are a timed-out run. await self._cancel_hook(definition, session) items = () status = TerminalRunStatus.TIMED_OUT diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 26eb4b3a..31dd5be9 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -9,11 +9,26 @@ from __future__ import annotations import ast +import builtins import hashlib +import os +import pickle import re +import select +import signal +import time +import warnings from collections.abc import Callable from typing import Any +try: + # Imported at MODULE level, never inside the forked child: acquiring the import + # lock in a child forked from a multi-threaded process is a classic fork + # deadlock. Absent on non-POSIX, where `_run_killable` degrades to a direct call. + import resource as _resource +except ImportError: # pragma: no cover - non-POSIX + _resource = None # type: ignore[assignment] + from failproofai_sdk.evaluator.authoring import ( Assertion, ConditionResult, @@ -25,6 +40,140 @@ MAX_CONDITION_SOURCE_BYTES = 16 * 1024 MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024 +# Static defense-in-depth bounds applied at COMPILE time (see `_compile`). They +# reject the obvious authoring bombs early; they are NOT the primary defense — +# a runtime-computed size (`range(len(session.events) ** 40)`) slips past any +# static check, which is exactly why the fork sandbox below is the real bound. +MAX_AST_NODES = 5_000 +MAX_POW_EXPONENT = 64 + +# Hard ceilings for ONE sandboxed evaluation, enforced by the kernel in a forked +# child (see `_run_killable`). CPU-seconds and wall-clock both bound compute +# bombs; RLIMIT_AS is the memory backstop for a giant-int / huge-allocation bomb. +DEFAULT_SANDBOX_TIMEOUT_SECONDS = 30 +SANDBOX_MEMORY_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB address space +SANDBOX_MAX_RESULT_BYTES = 4 * 1024 * 1024 # cap the pickled result read back + + +class EvaluationTimeout(Exception): + """A sandboxed evaluation exceeded its CPU/memory/wall-clock budget. + + Distinct from an eval that *returned* an error: the computation was forcibly + terminated because it could not be allowed to keep running. + """ + + +def _run_killable( + fn: Callable[[Any], Any], + session: Any, + *, + wall_timeout: float, + cpu_seconds: float, + mem_bytes: int, +) -> Any: + """Run ``fn(session)`` in a forked child that CANNOT outlive its budget. + + The child installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` and evaluates; the + parent waits at most ``wall_timeout`` and ``SIGKILL``s otherwise. This is the + one thing an in-process thread cannot do: cancelling a Python thread running + ``sum(range(10**20))`` leaves it burning CPU, but the kernel enforces these + limits on a separate process and the parent can kill it outright. Only the + result crosses back, over a pipe, as a small pickle. + """ + if not hasattr(os, "fork"): + # Non-POSIX (Windows) has no fork. The managed worker only ships on Linux + # containers, so this branch never runs in production; it keeps the SDK + # importable/testable elsewhere by degrading to a direct call. + return fn(session) + + read_fd, write_fd = os.pipe() + with warnings.catch_warnings(): + # The child is deliberately fork-safe — it acquires no lock any other + # thread holds (resource is imported at module level, pickle/os.write take + # no Python-level lock), then os._exit. Python 3.12's blanket + # "fork() in a multi-threaded process" DeprecationWarning does not apply. + warnings.simplefilter("ignore", DeprecationWarning) + pid = os.fork() + if pid == 0: # ---- child ---- + try: + os.close(read_fd) + try: + if _resource is not None: + cpu = max(1, int(cpu_seconds)) + _resource.setrlimit(_resource.RLIMIT_CPU, (cpu, cpu)) + _resource.setrlimit(_resource.RLIMIT_AS, (mem_bytes, mem_bytes)) + except Exception: # noqa: BLE001 - if limits can't be set, wall-clock still bounds it + pass + try: + payload = pickle.dumps(("ok", fn(session))) + except BaseException as error: # noqa: BLE001 - relay type+msg, never raise across the fork + payload = pickle.dumps( + ("err", type(error).__name__, str(error)[:500]) + ) + while payload: + written = os.write(write_fd, payload) + payload = payload[written:] + finally: + os._exit(0) # never run atexit / flush the parent's shared buffers + + # ---- parent ---- + os.close(write_fd) + chunks: list[bytes] = [] + timed_out = False + total = 0 + deadline = time.monotonic() + wall_timeout + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + ready, _, _ = select.select([read_fd], [], [], remaining) + if not ready: + timed_out = True + break + chunk = os.read(read_fd, 65536) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > SANDBOX_MAX_RESULT_BYTES: + timed_out = True # runaway output — treat as over-budget + break + finally: + os.close(read_fd) + if timed_out: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + if timed_out: + raise EvaluationTimeout("evaluation exceeded its CPU/memory/time budget") + data = b"".join(chunks) + if not data: + # Killed by RLIMIT_CPU/RLIMIT_AS before it could write a result. + raise EvaluationTimeout("evaluation was terminated before producing a result") + outcome = pickle.loads(data) + if outcome[0] == "ok": + return outcome[1] + # Re-raise with the child's original exception SEMANTICS preserved: an eval's + # `NameError`/`TypeError`/`ZeroDivisionError`/… and the sandbox's own + # `UnsafeEvaluatorSource` must read the same across the fork as they did + # in-process. Reconstruct any builtin exception by name; anything else + # collapses to a generic error — still caught as a failed run upstream. + _, name, message = outcome + if name == UnsafeEvaluatorSource.__name__: + raise UnsafeEvaluatorSource(message) + builtin = getattr(builtins, name, None) + if isinstance(builtin, type) and issubclass(builtin, BaseException): + raise builtin(message) + raise RuntimeError(f"{name}: {message}") + _ALLOWED_NODES = ( ast.Expression, ast.BoolOp, @@ -229,11 +378,33 @@ def _compile(source: str, *, field_name: str, maximum: int) -> Any: tree = ast.parse(source, mode="eval") except SyntaxError as error: raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + node_count = 0 for node in ast.walk(tree): + node_count += 1 + if node_count > MAX_AST_NODES: + raise UnsafeEvaluatorSource( + f"{field_name} is too large ({MAX_AST_NODES}-node ceiling)" + ) if not isinstance(node, _ALLOWED_NODES): raise UnsafeEvaluatorSource( f"{field_name} contains disallowed syntax: {type(node).__name__}" ) + # Defense in depth: a literal `10 ** 20` (or worse, `2 ** (10**8)`) builds a + # giant int — a memory bomb — at compile-time-visible size. Require Pow's + # exponent to be a small non-negative integer constant. Runtime-sized bombs + # still exist and are caught by the fork sandbox, not here. + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Pow): + exponent = node.right + if not ( + isinstance(exponent, ast.Constant) + and isinstance(exponent.value, int) + and not isinstance(exponent.value, bool) + and 0 <= exponent.value <= MAX_POW_EXPONENT + ): + raise UnsafeEvaluatorSource( + f"{field_name} exponent must be an integer constant " + f"in 0..{MAX_POW_EXPONENT}" + ) if isinstance(node, ast.Attribute): if node.attr.startswith("_"): raise UnsafeEvaluatorSource( @@ -268,33 +439,66 @@ def _forbid_object_reprs(field_name: str, value: Any) -> Any: return value -def compile_condition(source: str) -> Callable[[Any], bool | ConditionResult]: +def compile_condition( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, +) -> Callable[[Any], bool | ConditionResult]: code = _compile( source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES, ) + budget = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) - def condition(session: Any) -> bool | ConditionResult: + def _eval(session: Any) -> bool | ConditionResult: value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 if not isinstance(value, (bool, ConditionResult)): raise TypeError("condition_source must return bool or ConditionResult") return _forbid_object_reprs("condition_source", value) + def condition(session: Any) -> bool | ConditionResult: + # Managed conditions are sandboxed like evaluators — `sum(range(10**10)) > 0` + # in a condition would otherwise block the worker with NO timeout at all. + return _run_killable( + _eval, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + ) + return condition -def compile_evaluator(source: str) -> Callable[[Any], EvalResult]: +def compile_evaluator( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, +) -> Callable[[Any], EvalResult]: code = _compile( source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES, ) + budget = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) - def evaluate(session: Any) -> EvalResult: + def _eval(session: Any) -> EvalResult: value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 if not isinstance(value, EvalResult): raise TypeError("evaluator_source must return EvalResult") return _forbid_object_reprs("evaluator_source", value) + def evaluate(session: Any) -> EvalResult: + # The kernel-enforced boundary: this runs in a forked child with hard + # CPU/memory/wall-clock limits and is killed if it exceeds them, so a + # server-authored compute bomb cannot exhaust the worker (SEC-001). + return _run_killable( + _eval, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + ) + return evaluate diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py index 508d92ba..42591bb1 100644 --- a/sdk/python/tests/test_evaluator_http_e2e.py +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -222,7 +222,13 @@ def _plan(self, token: str, assignment_id: str, body: dict) -> None: if run["submission_id"] is None: run["worker_id"] = body["worker_id"] run["lease_generation"] = body["lease_generation"] - runs.append({"evaluation_run_id": run_id, **selected}) + runs.append( + { + "evaluation_run_id": run_id, + "execution_mode": "local", + **selected, + } + ) item["status"] = "planned" if runs else "skipped" self._json( 200, diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py index 99a874b6..e23b2474 100644 --- a/sdk/python/tests/test_evaluator_protocol.py +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -224,3 +224,23 @@ def test_session_helpers_use_the_protocol_event_vocabulary(): session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"]) assert session.count("tool_use") == 1 assert session.events_of_type("agent_end")[0].payload["summary"] == "Done" + + +def test_falsy_or_missing_execution_mode_is_rejected_not_defaulted(): + # F2: a falsy/absent execution_mode was silently coerced to 'local', which + # could run a server-authored ('python') definition down the customer path. + # It must now be a hard protocol error, not a default. + samples = _contract()["samples"] + for bad in ("", None): + defs = json.loads(json.dumps(samples["definitions_response"])) + defs["definitions"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(defs) + plan = json.loads(json.dumps(samples["plan_response"])) + plan["runs"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + PlanResponse.from_wire(plan) + absent = json.loads(json.dumps(samples["definitions_response"])) + del absent["definitions"][0]["execution_mode"] + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(absent) diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index c583319d..dc605101 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -4,7 +4,10 @@ from failproofai_sdk.evaluator import EvalResult, Score from failproofai_sdk.evaluator.source import ( + MAX_AST_NODES, MAX_EVALUATOR_SOURCE_BYTES, + MAX_POW_EXPONENT, + EvaluationTimeout, UnsafeEvaluatorSource, compile_condition, compile_evaluator, @@ -183,3 +186,56 @@ def test_source_checksum_covers_condition_and_evaluator_together(): assert base == source_checksum(None, "EvalResult()") assert base != source_checksum("True", "EvalResult()") assert base != source_checksum(None, "EvalResult(summary='changed')") + + +# --- SEC-001: managed source cannot exhaust the worker (killable-fork sandbox) --- + + +def test_compute_bomb_is_killed_within_its_budget(): + # `sum(range(10**9))` would burn CPU for ~20s in-process, uncancellable — a + # CPU-bound loop (not a big allocation) so the wall-clock/CPU budget is what + # stops it, deterministically, rather than the memory ceiling. The forked + # sandbox kills it at its budget. + import time as _time + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + started = _time.monotonic() + with pytest.raises(EvaluationTimeout): + evaluate(Session()) + assert _time.monotonic() - started < 5 # bounded by the ~1s budget, not ~20s + + +def test_condition_compute_bomb_is_also_bounded(): + condition = compile_condition("sum(range(10**8)) > 0", timeout_seconds=1) + with pytest.raises(EvaluationTimeout): + condition(Session()) + + +def test_literal_pow_exponent_bomb_is_rejected_at_compile(): + with pytest.raises(UnsafeEvaluatorSource, match="exponent"): + compile_evaluator(f"EvalResult(score=Score(10 ** {MAX_POW_EXPONENT + 1}))") + # A small constant exponent stays allowed. + result = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(2 ** 3))" + )(Session()) + assert result.reasoning == "8" + + +def test_oversized_expression_is_rejected_at_compile(): + huge = "[" + ",".join("1" for _ in range(MAX_AST_NODES)) + "]" + with pytest.raises(UnsafeEvaluatorSource, match="too large"): + compile_evaluator(f"EvalResult(score=Score(len({huge}) / len({huge})))") + + +def test_normal_managed_eval_survives_the_fork_boundary(): + # A session-dependent result must round-trip out of the forked child intact. + result = compile_evaluator( + "EvalResult(score=Score(1.0 if session.event_count > 0 else 0.0), " + "reasoning=str(session.event_count))" + )(Session()) + assert isinstance(result, EvalResult) + assert result.score.value == 1.0 + assert result.reasoning == "3" From 97ce938e984c02000c62a9d505f8794803f34c3a Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 16:17:01 +0530 Subject: [PATCH 08/20] fix(evaluator): fail closed when the fork sandbox is unavailable (hermes SEC-001) The non-POSIX fallback in _run_killable ran managed source directly (`return fn(session)`), so on a platform without os.fork an allowed-but-expensive expression like `sum(range(10**20))` got NO CPU/memory/wall-clock enforcement and could hold a worker slot indefinitely. Refuse instead: raise EvaluationSandboxUnavailable rather than execute server-authored source without a killable boundary. The managed worker only ships on Linux (fork present), so this never trips in production; it closes the "no fork => no sandbox => run it anyway" gap. New test simulates a fork-less platform and asserts managed eval + condition both fail closed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- .../failproofai_sdk/evaluator/source.py | 21 +++++++++++++++---- sdk/python/tests/test_evaluator_source.py | 14 +++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 31dd5be9..744b2aa5 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -63,6 +63,14 @@ class EvaluationTimeout(Exception): """ +class EvaluationSandboxUnavailable(Exception): + """The killable-process sandbox cannot be established on this platform. + + Raised instead of running server-authored source unsandboxed — without fork + there is no way to bound or terminate it, so we fail closed (SEC-001). + """ + + def _run_killable( fn: Callable[[Any], Any], session: Any, @@ -81,10 +89,15 @@ def _run_killable( result crosses back, over a pipe, as a small pickle. """ if not hasattr(os, "fork"): - # Non-POSIX (Windows) has no fork. The managed worker only ships on Linux - # containers, so this branch never runs in production; it keeps the SDK - # importable/testable elsewhere by degrading to a direct call. - return fn(session) + # Fail CLOSED. Without fork there is no way to bound or kill server-authored + # source, so refuse to run it rather than execute it unsandboxed — an + # allowed-but-expensive expression (`sum(range(10**20))`) would otherwise + # get NO CPU/memory/wall-clock enforcement. The managed worker only ships on + # Linux (fork present), so this never trips in production (SEC-001). + raise EvaluationSandboxUnavailable( + "managed evaluation requires a fork-based sandbox, " + "unavailable on this platform" + ) read_fd, write_fd = os.pipe() with warnings.catch_warnings(): diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index dc605101..90b139db 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -7,6 +7,7 @@ MAX_AST_NODES, MAX_EVALUATOR_SOURCE_BYTES, MAX_POW_EXPONENT, + EvaluationSandboxUnavailable, EvaluationTimeout, UnsafeEvaluatorSource, compile_condition, @@ -239,3 +240,16 @@ def test_normal_managed_eval_survives_the_fork_boundary(): assert isinstance(result, EvalResult) assert result.score.value == 1.0 assert result.reasoning == "3" + + +def test_managed_sandbox_fails_closed_when_fork_is_unavailable(monkeypatch): + # On a platform without os.fork there is no killable boundary, so managed + # source must be REFUSED, never run unsandboxed (SEC-001). Simulate by + # removing os.fork for the duration of the call. + import os as _os + + monkeypatch.delattr(_os, "fork", raising=False) + with pytest.raises(EvaluationSandboxUnavailable): + compile_evaluator("EvalResult(score=Score(1.0))")(Session()) + with pytest.raises(EvaluationSandboxUnavailable): + compile_condition("session.event_count > 0")(Session()) From bc96eda1e6552c1112c7bc5cdb9d61500f5fdbc8 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 16:41:12 +0530 Subject: [PATCH 09/20] fix(evaluator): fork+exec the sandbox + clamp the timeout (hermes SEC-001) Two SEC-001 problems in the fork-based sandbox: 1. os.fork() DEADLOCKS the worker. The managed worker is multi-threaded (asyncio loop, executor pool, writer daemon, health server); forking it and running Python in the child hangs on a lock another thread held at fork. Reproduced in the container: the worker registered, forked on its first managed eval, and hung (health down, no progress). Unit tests missed it because they fork from a single-threaded context. Replace fork-and-run-Python with fork+EXEC: a fresh `python -m failproofai_sdk.evaluator._sandbox_runner` process reads the (kind, source, transcript-wire, limits) tuple, installs RLIMIT_CPU + RLIMIT_AS on itself, evaluates, and returns the pickled result; the parent bounds wall-clock with subprocess timeout + kill. exec clears the inherited lock state, so it is safe from a multi-threaded process. Verified under real background-thread churn: normal eval works, compute + condition bombs are killed at budget, no hang. 2. The server-provided per-definition timeout had no upper bound, so a large timeout_seconds removed the execution bound. Clamp the effective CPU/wall budget to MAX_SANDBOX_TIMEOUT_SECONDS (60s). Fails closed (EvaluationSandboxUnavailable) if the sandbox cannot be spawned or the transcript is not serializable. Tests now use a real SessionTranscript (it must cross the process boundary via to_wire). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 29 +- .../evaluator/_sandbox_runner.py | 46 +++ .../failproofai_sdk/evaluator/source.py | 266 ++++++++---------- sdk/python/tests/test_evaluator_source.py | 60 ++-- 4 files changed, 231 insertions(+), 170 deletions(-) create mode 100644 sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 64619456..549a1bc5 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -36,17 +36,24 @@ it ships. dead-letters as one bounded `failed`/`eval_error` run instead of crashing the assignment task and forcing it to be reclaimed until its attempt budget runs out. -- Run managed (server-authored) evaluations in a **killable forked process** with - hard `RLIMIT_CPU` + `RLIMIT_AS` + wall-clock limits, killed on timeout — so a - compute/memory bomb in a hosted definition (`sum(range(10**20))`) can no longer - exhaust the worker (SEC-001). Cancelling an in-process thread does not stop it; - a forked process the kernel bounds and the parent can `SIGKILL` does. Managed - conditions, which previously ran with no timeout at all, are sandboxed the same - way. Defense in depth at compile time: reject `**` with a large/non-constant - exponent and cap total AST size. A managed condition the sandbox rejects now - dead-letters as `condition_error` instead of raising out of the plan loop and - stranding the assignment. Only server-authored source is isolated this way; - customer evaluators still run in-process (their own trusted code). +- Run managed (server-authored) evaluations in a **killable fork+exec'd + subprocess** with hard `RLIMIT_CPU` + `RLIMIT_AS` + a parent wall-clock kill — + so a compute/memory bomb in a hosted definition (`sum(range(10**20))`) can no + longer exhaust the worker (SEC-001). Cancelling an in-process thread does not + stop it; a fresh subprocess the kernel bounds and the parent terminates does. A + plain `os.fork()` would deadlock — the worker is multi-threaded (asyncio loop, + executor, writer) and forking one hangs the child on an inherited lock — so the + sandbox execs a fresh `python -m ..._sandbox_runner` that sets its own limits; + the transcript crosses in via `to_wire`, only the result crosses back. The + effective budget is **clamped to a hard ceiling** (`MAX_SANDBOX_TIMEOUT_SECONDS`, + 60s) so a large server-provided `timeout_seconds` cannot remove the bound. + Managed conditions, which previously ran with no timeout at all, are sandboxed + the same way. Fails **closed** (`EvaluationSandboxUnavailable`) if the sandbox + cannot be spawned or the transcript cannot be serialized. Defense in depth at + compile time: reject `**` with a large/non-constant exponent and cap total AST + size. A managed condition the sandbox rejects now dead-letters as + `condition_error` instead of stranding the assignment. Only server-authored + source is isolated this way; customer evaluators still run in-process. - Require `execution_mode` on the wire instead of coercing a falsy/missing value to `local` — a malformed value silently ran a `python` definition down the customer path (or vice-versa); it is now a hard protocol error. diff --git a/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py new file mode 100644 index 00000000..c8bf1636 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py @@ -0,0 +1,46 @@ +"""Subprocess entry point for the managed-source sandbox. + +Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner`` by +``source._run_sandboxed``. Reads a pickled +``(kind, source, session_wire, cpu_seconds, mem_bytes)`` tuple from stdin, +installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates the +re-validated server-authored source against the reconstructed transcript, and +writes a pickled ``("ok", result)`` / ``("err", type_name, message)`` outcome to +stdout. + +This is a FRESH exec'd process — never a fork of the multi-threaded worker — so +there is no inherited-lock deadlock (forking a process that has an asyncio loop, +an executor pool and a writer daemon hangs the child). The parent enforces the +wall-clock bound by killing this process on timeout. +""" + +from __future__ import annotations + +import pickle +import sys + + +def _main() -> int: + kind, source, session_wire, cpu_seconds, mem_bytes = pickle.loads( + sys.stdin.buffer.read() + ) + # Imported here, in the child, so the import cost is never on the worker's path. + from failproofai_sdk.evaluator.protocol import SessionTranscript + from failproofai_sdk.evaluator.source import _install_limits, _raw_eval + + try: + session = SessionTranscript.from_wire(session_wire) + # Compile + re-validate BEFORE the limits so validation cost is not charged + # against the eval's CPU budget; the limits bind the eval itself. + run = _raw_eval(source, kind) + _install_limits(cpu_seconds, mem_bytes) + out = pickle.dumps(("ok", run(session))) + except BaseException as error: # noqa: BLE001 - relay type+msg, this process is the boundary + out = pickle.dumps(("err", type(error).__name__, str(error)[:500])) + sys.stdout.buffer.write(out) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 744b2aa5..0676ad02 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -11,20 +11,14 @@ import ast import builtins import hashlib -import os import pickle import re -import select -import signal -import time -import warnings +import subprocess +import sys from collections.abc import Callable from typing import Any try: - # Imported at MODULE level, never inside the forked child: acquiring the import - # lock in a child forked from a multi-threaded process is a classic fork - # deadlock. Absent on non-POSIX, where `_run_killable` degrades to a direct call. import resource as _resource except ImportError: # pragma: no cover - non-POSIX _resource = None # type: ignore[assignment] @@ -47,12 +41,25 @@ MAX_AST_NODES = 5_000 MAX_POW_EXPONENT = 64 -# Hard ceilings for ONE sandboxed evaluation, enforced by the kernel in a forked -# child (see `_run_killable`). CPU-seconds and wall-clock both bound compute -# bombs; RLIMIT_AS is the memory backstop for a giant-int / huge-allocation bomb. +# Hard ceilings for ONE sandboxed evaluation, enforced by the kernel in a +# fork+exec'd subprocess (see `_run_sandboxed`). RLIMIT_CPU + the parent's +# wall-clock kill both bound compute bombs; RLIMIT_AS is the memory backstop for a +# giant-int / huge-allocation bomb. DEFAULT_SANDBOX_TIMEOUT_SECONDS = 30 +# The effective budget is CLAMPED to this ceiling regardless of the (server-set) +# per-definition timeout, so a large `timeout_seconds` can never remove the +# execution bound (SEC-001). Wall-clock and CPU are both capped here. +MAX_SANDBOX_TIMEOUT_SECONDS = 60 SANDBOX_MEMORY_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB address space -SANDBOX_MAX_RESULT_BYTES = 4 * 1024 * 1024 # cap the pickled result read back + + +def _clamp_budget(timeout_seconds: float | None) -> float: + """The wall-clock/CPU budget for one evaluation: a positive value no larger + than MAX_SANDBOX_TIMEOUT_SECONDS. Server-provided timeouts cannot exceed it.""" + requested = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) + if requested <= 0: + requested = DEFAULT_SANDBOX_TIMEOUT_SECONDS + return min(requested, float(MAX_SANDBOX_TIMEOUT_SECONDS)) class EvaluationTimeout(Exception): @@ -64,121 +71,80 @@ class EvaluationTimeout(Exception): class EvaluationSandboxUnavailable(Exception): - """The killable-process sandbox cannot be established on this platform. + """The killable-process sandbox could not be established. - Raised instead of running server-authored source unsandboxed — without fork - there is no way to bound or terminate it, so we fail closed (SEC-001). + Raised instead of running server-authored source unsandboxed — if the sandbox + subprocess cannot be started, or the transcript cannot be serialized into it, + there is no way to bound or terminate the evaluation, so we fail closed + (SEC-001). """ -def _run_killable( - fn: Callable[[Any], Any], +def _install_limits(cpu_seconds: float, mem_bytes: int) -> None: + """Install hard CPU + address-space limits on the CURRENT process. + + Called by the sandbox subprocess on itself, right before it evaluates. + """ + if _resource is None: # pragma: no cover - non-POSIX + return + cpu = max(1, int(cpu_seconds)) + _resource.setrlimit(_resource.RLIMIT_CPU, (cpu, cpu)) + _resource.setrlimit(_resource.RLIMIT_AS, (mem_bytes, mem_bytes)) + + +def _run_sandboxed( + kind: str, + source: str, session: Any, *, wall_timeout: float, cpu_seconds: float, mem_bytes: int, ) -> Any: - """Run ``fn(session)`` in a forked child that CANNOT outlive its budget. - - The child installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` and evaluates; the - parent waits at most ``wall_timeout`` and ``SIGKILL``s otherwise. This is the - one thing an in-process thread cannot do: cancelling a Python thread running - ``sum(range(10**20))`` leaves it burning CPU, but the kernel enforces these - limits on a separate process and the parent can kill it outright. Only the - result crosses back, over a pipe, as a small pickle. + """Evaluate server-authored ``source`` against ``session`` in a fork+exec'd + subprocess that CANNOT outlive its budget. + + A FRESH ``python -m ..._sandbox_runner`` process — never a fork of this + multi-threaded worker (forking one deadlocks the child on a lock some other + thread holds) — installs hard RLIMIT_CPU + RLIMIT_AS on itself and evaluates; + this parent kills it after ``wall_timeout``. So an in-process thread running + ``sum(range(10**20))`` that cannot be cancelled becomes a separate process the + kernel bounds and the parent terminates outright. The transcript crosses in as + its wire dict; only the result crosses back, as a small pickle. """ - if not hasattr(os, "fork"): - # Fail CLOSED. Without fork there is no way to bound or kill server-authored - # source, so refuse to run it rather than execute it unsandboxed — an - # allowed-but-expensive expression (`sum(range(10**20))`) would otherwise - # get NO CPU/memory/wall-clock enforcement. The managed worker only ships on - # Linux (fork present), so this never trips in production (SEC-001). + try: + session_wire = session.to_wire() + except AttributeError as error: raise EvaluationSandboxUnavailable( - "managed evaluation requires a fork-based sandbox, " - "unavailable on this platform" - ) - - read_fd, write_fd = os.pipe() - with warnings.catch_warnings(): - # The child is deliberately fork-safe — it acquires no lock any other - # thread holds (resource is imported at module level, pickle/os.write take - # no Python-level lock), then os._exit. Python 3.12's blanket - # "fork() in a multi-threaded process" DeprecationWarning does not apply. - warnings.simplefilter("ignore", DeprecationWarning) - pid = os.fork() - if pid == 0: # ---- child ---- - try: - os.close(read_fd) - try: - if _resource is not None: - cpu = max(1, int(cpu_seconds)) - _resource.setrlimit(_resource.RLIMIT_CPU, (cpu, cpu)) - _resource.setrlimit(_resource.RLIMIT_AS, (mem_bytes, mem_bytes)) - except Exception: # noqa: BLE001 - if limits can't be set, wall-clock still bounds it - pass - try: - payload = pickle.dumps(("ok", fn(session))) - except BaseException as error: # noqa: BLE001 - relay type+msg, never raise across the fork - payload = pickle.dumps( - ("err", type(error).__name__, str(error)[:500]) - ) - while payload: - written = os.write(write_fd, payload) - payload = payload[written:] - finally: - os._exit(0) # never run atexit / flush the parent's shared buffers - - # ---- parent ---- - os.close(write_fd) - chunks: list[bytes] = [] - timed_out = False - total = 0 - deadline = time.monotonic() + wall_timeout + "sandboxed evaluation requires a serializable transcript" + ) from error + payload = pickle.dumps((kind, source, session_wire, cpu_seconds, mem_bytes)) try: - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - break - ready, _, _ = select.select([read_fd], [], [], remaining) - if not ready: - timed_out = True - break - chunk = os.read(read_fd, 65536) - if not chunk: - break - chunks.append(chunk) - total += len(chunk) - if total > SANDBOX_MAX_RESULT_BYTES: - timed_out = True # runaway output — treat as over-budget - break - finally: - os.close(read_fd) - if timed_out: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - try: - os.waitpid(pid, 0) - except ChildProcessError: - pass - - if timed_out: - raise EvaluationTimeout("evaluation exceeded its CPU/memory/time budget") - data = b"".join(chunks) - if not data: - # Killed by RLIMIT_CPU/RLIMIT_AS before it could write a result. + completed = subprocess.run( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner"], + input=payload, + capture_output=True, + timeout=wall_timeout, + ) + except subprocess.TimeoutExpired as error: + # subprocess.run has already SIGKILLed the process on the way out. + raise EvaluationTimeout("evaluation exceeded its wall-clock budget") from error + except OSError as error: + # Could not even spawn the sandbox — fail closed rather than run unbounded. + raise EvaluationSandboxUnavailable( + f"could not start the evaluation sandbox: {error}" + ) from error + if completed.returncode != 0 or not completed.stdout: + # Killed by RLIMIT_CPU/RLIMIT_AS (or otherwise died) before it could write. raise EvaluationTimeout("evaluation was terminated before producing a result") - outcome = pickle.loads(data) + outcome = pickle.loads(completed.stdout) if outcome[0] == "ok": return outcome[1] - # Re-raise with the child's original exception SEMANTICS preserved: an eval's - # `NameError`/`TypeError`/`ZeroDivisionError`/… and the sandbox's own - # `UnsafeEvaluatorSource` must read the same across the fork as they did - # in-process. Reconstruct any builtin exception by name; anything else - # collapses to a generic error — still caught as a failed run upstream. + # Preserve the child's original exception SEMANTICS: an eval's + # NameError/TypeError/ZeroDivisionError/... and the sandbox's own + # UnsafeEvaluatorSource must read the same as they did in-process. Reconstruct + # any builtin exception by name; anything else collapses to a generic error — + # still caught as a failed run upstream. _, name, message = outcome if name == UnsafeEvaluatorSource.__name__: raise UnsafeEvaluatorSource(message) @@ -452,29 +418,56 @@ def _forbid_object_reprs(field_name: str, value: Any) -> Any: return value +def _raw_eval(source: str, kind: str) -> Callable[[Any], Any]: + """Compile server-authored source and return a function that evaluates it and + validates the result. + + Runs INSIDE the sandbox subprocess (see `_sandbox_runner`) — there is no + isolation here. `compile_condition`/`compile_evaluator` have already validated + the AST in the parent; this recompiles as defense in depth so a subprocess + can never eval source the parent has not vetted. + """ + if kind == "condition": + code = _compile( + source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 + if not isinstance(value, (bool, ConditionResult)): + raise TypeError("condition_source must return bool or ConditionResult") + return _forbid_object_reprs("condition_source", value) + + else: + code = _compile( + source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 + if not isinstance(value, EvalResult): + raise TypeError("evaluator_source must return EvalResult") + return _forbid_object_reprs("evaluator_source", value) + + return run + + def compile_condition( source: str, *, timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, ) -> Callable[[Any], bool | ConditionResult]: - code = _compile( - source, - field_name="condition_source", - maximum=MAX_CONDITION_SOURCE_BYTES, - ) - budget = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) - - def _eval(session: Any) -> bool | ConditionResult: - value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 - if not isinstance(value, (bool, ConditionResult)): - raise TypeError("condition_source must return bool or ConditionResult") - return _forbid_object_reprs("condition_source", value) + # Validate the AST in THIS (parent) process so unsafe/malformed source is + # rejected up front, before any subprocess is spawned. + _compile(source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) def condition(session: Any) -> bool | ConditionResult: # Managed conditions are sandboxed like evaluators — `sum(range(10**10)) > 0` # in a condition would otherwise block the worker with NO timeout at all. - return _run_killable( - _eval, + return _run_sandboxed( + "condition", + source, session, wall_timeout=budget, cpu_seconds=budget, @@ -489,25 +482,16 @@ def compile_evaluator( *, timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, ) -> Callable[[Any], EvalResult]: - code = _compile( - source, - field_name="evaluator_source", - maximum=MAX_EVALUATOR_SOURCE_BYTES, - ) - budget = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) - - def _eval(session: Any) -> EvalResult: - value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 - if not isinstance(value, EvalResult): - raise TypeError("evaluator_source must return EvalResult") - return _forbid_object_reprs("evaluator_source", value) + _compile(source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) def evaluate(session: Any) -> EvalResult: - # The kernel-enforced boundary: this runs in a forked child with hard - # CPU/memory/wall-clock limits and is killed if it exceeds them, so a + # The kernel-enforced boundary: this runs in a fork+exec'd subprocess with + # hard CPU/memory/wall-clock limits, killed if it exceeds them, so a # server-authored compute bomb cannot exhaust the worker (SEC-001). - return _run_killable( - _eval, + return _run_sandboxed( + "evaluator", + source, session, wall_timeout=budget, cpu_seconds=budget, diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 90b139db..bf26acd9 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -3,21 +3,46 @@ import pytest from failproofai_sdk.evaluator import EvalResult, Score +from failproofai_sdk.evaluator.protocol import SessionTranscript, TranscriptEvent from failproofai_sdk.evaluator.source import ( MAX_AST_NODES, MAX_EVALUATOR_SOURCE_BYTES, MAX_POW_EXPONENT, + MAX_SANDBOX_TIMEOUT_SECONDS, EvaluationSandboxUnavailable, EvaluationTimeout, UnsafeEvaluatorSource, + _clamp_budget, compile_condition, compile_evaluator, source_checksum, ) -class Session: - event_count = 3 +def Session(event_count: int = 3) -> SessionTranscript: + """A real, serializable transcript — managed evals now run in a subprocess and + the transcript crosses the boundary via `to_wire`, so a dummy object won't do. + Each event carries a dict payload (so `events[0].payload.get` is reachable).""" + events = tuple( + TranscriptEvent( + id=f"e{i}", + ts="2026-08-28T12:00:00.000000Z", + event_type="tool_use", + payload={"k": "v", "tool_name": "search"}, + ) + for i in range(event_count) + ) + return SessionTranscript( + assignment_id="a", + session_id="s", + session_revision_id="r", + agent_id="agent", + environment="test", + started_at="2026-08-28T12:00:00.000000Z", + ended_at="2026-08-28T12:00:01.000000Z", + event_count=event_count, + events=events, + ) def test_restricted_expressions_can_evaluate_conditions_and_results(): @@ -126,12 +151,6 @@ def test_no_reachable_construct_leaks_a_heap_pointer_repr(): ], ) def test_object_repr_pointer_disclosure_is_rejected_at_the_output(inner): - class Sess: - class _E: - payload = {"k": "v"} - - events = (_E(),) - for field in ( f'EvalResult(score=Score(1.0), reasoning=str({inner}))', f'EvalResult(score=Score(1.0), summary=str({inner}))', @@ -140,7 +159,7 @@ class _E: ): evaluate = compile_evaluator(field) with pytest.raises(UnsafeEvaluatorSource, match="object repr"): - evaluate(Sess()) + evaluate(Session()) def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next(): @@ -242,14 +261,19 @@ def test_normal_managed_eval_survives_the_fork_boundary(): assert result.reasoning == "3" -def test_managed_sandbox_fails_closed_when_fork_is_unavailable(monkeypatch): - # On a platform without os.fork there is no killable boundary, so managed - # source must be REFUSED, never run unsandboxed (SEC-001). Simulate by - # removing os.fork for the duration of the call. - import os as _os +def test_sandbox_fails_closed_without_a_serializable_transcript(): + # The transcript crosses into the subprocess via `to_wire`. A session that + # can't be serialized cannot be sandboxed, so refuse rather than run unbounded. + class NotATranscript: + event_count = 1 - monkeypatch.delattr(_os, "fork", raising=False) with pytest.raises(EvaluationSandboxUnavailable): - compile_evaluator("EvalResult(score=Score(1.0))")(Session()) - with pytest.raises(EvaluationSandboxUnavailable): - compile_condition("session.event_count > 0")(Session()) + compile_evaluator("EvalResult(score=Score(1.0))")(NotATranscript()) + + +def test_server_timeout_cannot_exceed_the_hard_ceiling(): + # SEC-001: a large server-provided timeout must not remove the execution bound. + assert _clamp_budget(10**9) == float(MAX_SANDBOX_TIMEOUT_SECONDS) + assert _clamp_budget(0) == 30.0 + assert _clamp_budget(None) == 30.0 + assert _clamp_budget(5) == 5.0 From 70b522474a985740acd6f8a14b59121e2ba2c4f3 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 17:02:02 +0530 Subject: [PATCH 10/20] fix(evaluator): bound the sandbox result on both sides (hermes SEC-001) The parent read the sandbox child's entire stdout and pickle.loads'd it, so a permitted expression building a huge result (metrics={str(x):1 for x in range(100000)}) could OOM the worker despite the child RLIMIT. Bound it on both sides: the child now validates the result (result_items / the 25-result limit) and refuses to serialize anything over SANDBOX_MAX_RESULT_BYTES (1 MiB) before it crosses; the parent reads via a temp-file-in / Popen with a capped, timed stdout read and kills the child on overflow or timeout. Input moves to a temp file (the transcript can be large; feeding a big stdin while bounding stdout invites a pipe deadlock). eval_key is threaded to the sandbox so the child can enforce the 25-item limit. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 8 +- .../evaluator/_sandbox_runner.py | 47 +++++--- .../failproofai_sdk/evaluator/runtime.py | 7 +- .../failproofai_sdk/evaluator/source.py | 112 ++++++++++++++---- sdk/python/tests/test_evaluator_source.py | 12 ++ 5 files changed, 142 insertions(+), 44 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 549a1bc5..2053f5b1 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -48,8 +48,12 @@ it ships. effective budget is **clamped to a hard ceiling** (`MAX_SANDBOX_TIMEOUT_SECONDS`, 60s) so a large server-provided `timeout_seconds` cannot remove the bound. Managed conditions, which previously ran with no timeout at all, are sandboxed - the same way. Fails **closed** (`EvaluationSandboxUnavailable`) if the sandbox - cannot be spawned or the transcript cannot be serialized. Defense in depth at + the same way. The result crossing back is **bounded on both sides** — the child + validates it (`result_items`, the 25-result limit) and refuses to serialize + anything over 1 MiB, and the parent reads at most that before killing the child + — so an oversized result (`metrics={str(x): 1 for x in range(100000)}`) cannot + OOM the worker either. Fails **closed** (`EvaluationSandboxUnavailable`) if the + sandbox cannot be spawned or the transcript cannot be serialized. Defense in depth at compile time: reject `**` with a large/non-constant exponent and cap total AST size. A managed condition the sandbox rejects now dead-letters as `condition_error` instead of stranding the assignment. Only server-authored diff --git a/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py index c8bf1636..1b773e26 100644 --- a/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py +++ b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py @@ -1,17 +1,18 @@ """Subprocess entry point for the managed-source sandbox. -Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner`` by -``source._run_sandboxed``. Reads a pickled -``(kind, source, session_wire, cpu_seconds, mem_bytes)`` tuple from stdin, -installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates the -re-validated server-authored source against the reconstructed transcript, and -writes a pickled ``("ok", result)`` / ``("err", type_name, message)`` outcome to -stdout. +Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner `` by +``source._run_sandboxed``. The input file holds a pickled +``(kind, source, session_wire, cpu_seconds, mem_bytes, eval_key)`` tuple. This +process installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates +the re-validated server-authored source against the reconstructed transcript, +validates + bounds the result, and writes a pickled ``("ok", result)`` / +``("err", type_name, message)`` outcome to stdout. This is a FRESH exec'd process — never a fork of the multi-threaded worker — so there is no inherited-lock deadlock (forking a process that has an asyncio loop, an executor pool and a writer daemon hangs the child). The parent enforces the -wall-clock bound by killing this process on timeout. +wall-clock bound and the output-size bound by reading only so far and killing +this process on timeout or overflow. """ from __future__ import annotations @@ -21,12 +22,17 @@ def _main() -> int: - kind, source, session_wire, cpu_seconds, mem_bytes = pickle.loads( - sys.stdin.buffer.read() - ) + with open(sys.argv[1], "rb") as handle: + kind, source, session_wire, cpu_seconds, mem_bytes, eval_key = pickle.loads( + handle.read() + ) # Imported here, in the child, so the import cost is never on the worker's path. from failproofai_sdk.evaluator.protocol import SessionTranscript - from failproofai_sdk.evaluator.source import _install_limits, _raw_eval + from failproofai_sdk.evaluator.source import ( + SANDBOX_MAX_RESULT_BYTES, + _install_limits, + _raw_eval, + ) try: session = SessionTranscript.from_wire(session_wire) @@ -34,10 +40,21 @@ def _main() -> int: # against the eval's CPU budget; the limits bind the eval itself. run = _raw_eval(source, kind) _install_limits(cpu_seconds, mem_bytes) - out = pickle.dumps(("ok", run(session))) + result = run(session) + # Bound the result INSIDE the sandbox before it crosses back: result_items + # enforces the 25-result limit + field validation, so a huge result + # (`metrics={str(x):1 for x in range(100000)}`) raises here instead of + # being serialized and shipped to the parent. + if kind == "evaluator": + result.result_items(eval_key or "result") + payload = pickle.dumps(("ok", result)) + if len(payload) > SANDBOX_MAX_RESULT_BYTES: + payload = pickle.dumps( + ("err", "ResultTooLarge", "evaluation result exceeds the size limit") + ) except BaseException as error: # noqa: BLE001 - relay type+msg, this process is the boundary - out = pickle.dumps(("err", type(error).__name__, str(error)[:500])) - sys.stdout.buffer.write(out) + payload = pickle.dumps(("err", type(error).__name__, str(error)[:500])) + sys.stdout.buffer.write(payload) sys.stdout.buffer.flush() return 0 diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index c9f89b5b..01abe9c5 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -58,7 +58,7 @@ def _utc_now() -> str: ) -def _deferred_managed_eval(source: str, timeout_seconds: int | None): +def _deferred_managed_eval(source: str, timeout_seconds: int | None, eval_key: str): """Compile server-authored source lazily, at invocation time. Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``). @@ -71,7 +71,9 @@ def _deferred_managed_eval(source: str, timeout_seconds: int | None): """ def evaluate(session: Any) -> Any: - return compile_evaluator(source, timeout_seconds=timeout_seconds)(session) + return compile_evaluator( + source, timeout_seconds=timeout_seconds, eval_key=eval_key + )(session) return evaluate @@ -440,6 +442,7 @@ async def process_assignment(self, assignment: Assignment) -> None: function=_deferred_managed_eval( run.evaluator_source, run.timeout_seconds or descriptor.timeout_seconds, + descriptor.eval_key, ), condition=None, on_cancel=None, diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 0676ad02..1525b1d9 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -11,10 +11,14 @@ import ast import builtins import hashlib +import os import pickle import re +import select import subprocess import sys +import tempfile +import time from collections.abc import Callable from typing import Any @@ -51,6 +55,13 @@ # execution bound (SEC-001). Wall-clock and CPU are both capped here. MAX_SANDBOX_TIMEOUT_SECONDS = 60 SANDBOX_MEMORY_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB address space +# The result crossing back is bounded on BOTH sides: the child refuses to serialize +# a result larger than this, and the parent stops reading (and kills the child) +# past it — so a permitted expression that builds a huge result +# (`EvalResult(metrics={str(x): 1 for x in range(100000)})`) cannot OOM the worker +# even though the child's RLIMIT_AS lets it construct one. A valid result (<=25 +# items, bounded fields) is far under this. +SANDBOX_MAX_RESULT_BYTES = 1 * 1024 * 1024 # 1 MiB def _clamp_budget(timeout_seconds: float | None) -> float: @@ -100,17 +111,19 @@ def _run_sandboxed( wall_timeout: float, cpu_seconds: float, mem_bytes: int, + eval_key: str | None = None, ) -> Any: """Evaluate server-authored ``source`` against ``session`` in a fork+exec'd - subprocess that CANNOT outlive its budget. + subprocess that CANNOT outlive its budget or flood this process. A FRESH ``python -m ..._sandbox_runner`` process — never a fork of this multi-threaded worker (forking one deadlocks the child on a lock some other - thread holds) — installs hard RLIMIT_CPU + RLIMIT_AS on itself and evaluates; - this parent kills it after ``wall_timeout``. So an in-process thread running - ``sum(range(10**20))`` that cannot be cancelled becomes a separate process the - kernel bounds and the parent terminates outright. The transcript crosses in as - its wire dict; only the result crosses back, as a small pickle. + thread holds) — reads its input from a temp file, installs hard RLIMIT_CPU + + RLIMIT_AS on itself, evaluates, and writes a bounded result to stdout. This + parent reads stdout up to ``SANDBOX_MAX_RESULT_BYTES`` and no further, killing + the child on timeout OR oversize — so neither compute (``sum(range(10**20))``) + nor an oversized result (``metrics={str(x):1 for x in range(100000)}``) can + exhaust the worker. """ try: session_wire = session.to_wire() @@ -118,26 +131,71 @@ def _run_sandboxed( raise EvaluationSandboxUnavailable( "sandboxed evaluation requires a serializable transcript" ) from error - payload = pickle.dumps((kind, source, session_wire, cpu_seconds, mem_bytes)) + payload = pickle.dumps( + (kind, source, session_wire, cpu_seconds, mem_bytes, eval_key) + ) + # Input via a temp file, not stdin: the transcript can be large (up to the + # transcript ceiling) and feeding a big stdin while bounding stdout invites a + # pipe deadlock. The child reads the file; we only read its stdout. + handle, path = tempfile.mkstemp(prefix="fpai-sandbox-", suffix=".pkl") try: - completed = subprocess.run( # noqa: S603 - fixed argv, no shell - [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner"], - input=payload, - capture_output=True, - timeout=wall_timeout, - ) - except subprocess.TimeoutExpired as error: - # subprocess.run has already SIGKILLed the process on the way out. - raise EvaluationTimeout("evaluation exceeded its wall-clock budget") from error - except OSError as error: - # Could not even spawn the sandbox — fail closed rather than run unbounded. - raise EvaluationSandboxUnavailable( - f"could not start the evaluation sandbox: {error}" - ) from error - if completed.returncode != 0 or not completed.stdout: + with os.fdopen(handle, "wb") as tmp: + tmp.write(payload) + try: + proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError as error: + raise EvaluationSandboxUnavailable( + f"could not start the evaluation sandbox: {error}" + ) from error + chunks: list[bytes] = [] + total = 0 + timed_out = False + too_large = False + deadline = time.monotonic() + wall_timeout + out_fd = proc.stdout.fileno() + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + ready, _, _ = select.select([out_fd], [], [], remaining) + if not ready: + timed_out = True + break + chunk = os.read(out_fd, 65536) + if not chunk: + break + total += len(chunk) + if total > SANDBOX_MAX_RESULT_BYTES: + too_large = True + break + chunks.append(chunk) + finally: + proc.stdout.close() + if proc.poll() is None: + proc.kill() + proc.wait() + finally: + try: + os.unlink(path) + except OSError: + pass + + if timed_out: + raise EvaluationTimeout("evaluation exceeded its wall-clock budget") + if too_large: + raise EvaluationTimeout("evaluation result exceeded the size limit") + data = b"".join(chunks) + if not data: # Killed by RLIMIT_CPU/RLIMIT_AS (or otherwise died) before it could write. raise EvaluationTimeout("evaluation was terminated before producing a result") - outcome = pickle.loads(completed.stdout) + outcome = pickle.loads(data) if outcome[0] == "ok": return outcome[1] # Preserve the child's original exception SEMANTICS: an eval's @@ -481,14 +539,17 @@ def compile_evaluator( source: str, *, timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, + eval_key: str | None = None, ) -> Callable[[Any], EvalResult]: _compile(source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES) budget = _clamp_budget(timeout_seconds) def evaluate(session: Any) -> EvalResult: # The kernel-enforced boundary: this runs in a fork+exec'd subprocess with - # hard CPU/memory/wall-clock limits, killed if it exceeds them, so a - # server-authored compute bomb cannot exhaust the worker (SEC-001). + # hard CPU/memory/wall-clock limits and a bounded result, killed if it + # exceeds them, so a server-authored compute or result bomb cannot exhaust + # the worker (SEC-001). `eval_key` lets the child validate the result's + # 25-item limit before it crosses back. return _run_sandboxed( "evaluator", source, @@ -496,6 +557,7 @@ def evaluate(session: Any) -> EvalResult: wall_timeout=budget, cpu_seconds=budget, mem_bytes=SANDBOX_MEMORY_BYTES, + eval_key=eval_key, ) return evaluate diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index bf26acd9..37bf7323 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -277,3 +277,15 @@ def test_server_timeout_cannot_exceed_the_hard_ceiling(): assert _clamp_budget(0) == 30.0 assert _clamp_budget(None) == 30.0 assert _clamp_budget(5) == 5.0 + + +def test_oversized_result_is_rejected_before_it_crosses_back(): + # A result with far more than the 25-item limit must be rejected INSIDE the + # sandbox (via result_items), so a huge result can never be serialized and + # shipped back to OOM the worker (SEC-001). + src = ( + "EvalResult(score=Score(1.0), " + "metrics={'m' + str(i): float(i) for i in range(200)})" + ) + with pytest.raises(ValueError, match="at most"): + compile_evaluator(src, eval_key="q")(Session()) From db6bd9e6ba24083e03ad0a69e0eac70b645b10de Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 17:31:45 +0530 Subject: [PATCH 11/20] fix(evaluator): bound aggregate sandbox memory (hermes SEC-001) A 2 GiB per-sandbox limit did not bound the host: max_concurrency up to 32 could run 32 sandboxes at once (~64 GiB), and a permitted expression can allocate memory before returning a valid result (`([0]*200000000, EvalResult(...))[1]`). Lower the per-sandbox address space to 512 MiB (generous for a <=25 MiB transcript, rejects the ~1.6 GiB allocation) AND cap concurrent sandbox processes with a semaphore, so the aggregate (~2 GiB) is bounded independent of the worker's claim concurrency. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 2 +- .../failproofai_sdk/evaluator/source.py | 89 +++++++++++-------- sdk/python/tests/test_evaluator_source.py | 9 ++ 3 files changed, 62 insertions(+), 38 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 2053f5b1..ad9b9c63 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -52,7 +52,7 @@ it ships. validates it (`result_items`, the 25-result limit) and refuses to serialize anything over 1 MiB, and the parent reads at most that before killing the child — so an oversized result (`metrics={str(x): 1 for x in range(100000)}`) cannot - OOM the worker either. Fails **closed** (`EvaluationSandboxUnavailable`) if the + OOM the worker either. The per-sandbox address space is capped (512 MiB) and the number of concurrent sandbox processes is bounded (a semaphore), so the AGGREGATE memory is bounded independent of the worker's `max_concurrency` — a fleet of concurrent runs can't OOM the host. Fails **closed** (`EvaluationSandboxUnavailable`) if the sandbox cannot be spawned or the transcript cannot be serialized. Defense in depth at compile time: reject `**` with a large/non-constant exponent and cap total AST size. A managed condition the sandbox rejects now dead-letters as diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 1525b1d9..69e24bef 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -18,6 +18,7 @@ import subprocess import sys import tempfile +import threading import time from collections.abc import Callable from typing import Any @@ -54,7 +55,17 @@ # per-definition timeout, so a large `timeout_seconds` can never remove the # execution bound (SEC-001). Wall-clock and CPU are both capped here. MAX_SANDBOX_TIMEOUT_SECONDS = 60 -SANDBOX_MEMORY_BYTES = 2 * 1024 * 1024 * 1024 # 2 GiB address space +# Per-sandbox address-space cap. A managed eval works over a transcript (<=25 MiB) +# and returns a small result, so this is generous; it also rejects an allocation +# bomb (`[0] * 200000000` is ~1.6 GiB > this) before it returns a valid result. +SANDBOX_MEMORY_BYTES = 512 * 1024 * 1024 # 512 MiB +# ...but a per-process cap alone does not bound the HOST: a worker with +# max_concurrency=32 could run 32 sandboxes at once. Cap the number of concurrent +# sandbox processes so the AGGREGATE (MAX_CONCURRENT_SANDBOXES * SANDBOX_MEMORY_BYTES, +# ~2 GiB) is bounded independent of the worker's claim concurrency; extra evals +# queue on the semaphore rather than pile up memory. +MAX_CONCURRENT_SANDBOXES = 4 +_SANDBOX_SLOTS = threading.Semaphore(MAX_CONCURRENT_SANDBOXES) # The result crossing back is bounded on BOTH sides: the child refuses to serialize # a result larger than this, and the parent stops reading (and kills the child) # past it — so a permitted expression that builds a huge result @@ -141,46 +152,50 @@ def _run_sandboxed( try: with os.fdopen(handle, "wb") as tmp: tmp.write(payload) - try: - proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell - [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) - except OSError as error: - raise EvaluationSandboxUnavailable( - f"could not start the evaluation sandbox: {error}" - ) from error chunks: list[bytes] = [] total = 0 timed_out = False too_large = False - deadline = time.monotonic() + wall_timeout - out_fd = proc.stdout.fileno() - try: - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - break - ready, _, _ = select.select([out_fd], [], [], remaining) - if not ready: - timed_out = True - break - chunk = os.read(out_fd, 65536) - if not chunk: - break - total += len(chunk) - if total > SANDBOX_MAX_RESULT_BYTES: - too_large = True - break - chunks.append(chunk) - finally: - proc.stdout.close() - if proc.poll() is None: - proc.kill() - proc.wait() + # Hold a slot for the whole subprocess lifetime so no more than + # MAX_CONCURRENT_SANDBOXES run at once — bounds the aggregate memory the + # sandboxes can consume regardless of the worker's claim concurrency. + with _SANDBOX_SLOTS: + try: + proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError as error: + raise EvaluationSandboxUnavailable( + f"could not start the evaluation sandbox: {error}" + ) from error + deadline = time.monotonic() + wall_timeout + out_fd = proc.stdout.fileno() + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + ready, _, _ = select.select([out_fd], [], [], remaining) + if not ready: + timed_out = True + break + chunk = os.read(out_fd, 65536) + if not chunk: + break + total += len(chunk) + if total > SANDBOX_MAX_RESULT_BYTES: + too_large = True + break + chunks.append(chunk) + finally: + proc.stdout.close() + if proc.poll() is None: + proc.kill() + proc.wait() finally: try: os.unlink(path) diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 37bf7323..897603dc 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -289,3 +289,12 @@ def test_oversized_result_is_rejected_before_it_crosses_back(): ) with pytest.raises(ValueError, match="at most"): compile_evaluator(src, eval_key="q")(Session()) + + +def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit(): + # A ~1.6 GiB allocation exceeds the per-sandbox RLIMIT_AS and is killed, so it + # cannot exhaust the worker even wrapped in an otherwise-valid result. With the + # concurrent-sandbox cap this also bounds the aggregate across concurrent runs. + src = "EvalResult(score=Score(1.0 if len([0] * 200000000) >= 0 else 0.0))" + with pytest.raises((EvaluationTimeout, MemoryError)): + compile_evaluator(src, timeout_seconds=5)(Session()) From fb13990cd699c8e787560dcbd6e13a1de88378c3 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 18:27:14 +0530 Subject: [PATCH 12/20] Harden evaluator v2 sandbox; fix condition selection and error mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - COR-001: managed (python) applicability now follows the server's condition_source, never a colliding local condition with the same (eval_key, eval_version). Condition selection branches on execution_mode, mirroring the evaluator branch below it. - API-001: recognize the server's `incomplete_plan` terminal error — added to the ERROR_SPECS mirror and the shared contract.json fixture (byte-identical with the server). - Adversarial-audit (SEC): close a heap-address disclosure bypass. The output-boundary `` guard was anchored on `<`, so an allow-listed str(x).replace("<","") / f-string / % kept the live address while stripping the match. The defense moves to compile time: a bound method (the only reachable value with a pointer repr — transcript and result types are frozen, pointer-free dataclasses) may only be CALLED, never referenced as a bare value, so no reachable value carries a pointer repr through str()/f-string/%. The output-boundary scan is kept and broadened (no leading `<`) as defense in depth. Regression tests cover the colliding-key condition, the three bypass payloads, and that legitimate called-method/data-attribute stringification still works. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 31 ++++++++++ .../failproofai_sdk/evaluator/protocol.py | 1 + .../failproofai_sdk/evaluator/runtime.py | 28 +++++---- .../failproofai_sdk/evaluator/source.py | 59 +++++++++++++++---- .../tests/fixtures/evaluator_v2/contract.json | 1 + sdk/python/tests/test_evaluator_runtime.py | 53 +++++++++++++++++ sdk/python/tests/test_evaluator_source.py | 54 ++++++++++++++--- 7 files changed, 196 insertions(+), 31 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index ad9b9c63..421afd53 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -61,6 +61,37 @@ it ships. - Require `execution_mode` on the wire instead of coercing a falsy/missing value to `local` — a malformed value silently ran a `python` definition down the customer path (or vice-versa); it is now a hard protocol error. +- A managed (`python`) definition's applicability is now governed by the SERVER's + `condition_source`, never a colliding local condition (COR-001). `process_assignment` + keyed the local-definition lookup on `(eval_key, eval_version)` alone and selected + `local.condition` whenever a local definition with that key existed — so a managed + definition whose server condition was false could be forced to run anyway if the + worker had also registered a local definition under the same key whose condition was + true, executing server-managed source against the operator's intent. Condition + selection now branches on `execution_mode`, mirroring the evaluator branch: `LOCAL` + uses `local.condition`, `PYTHON` compiles and runs the server's `condition_source` + regardless of any key collision. Regression test: identical local+managed keys, local + condition true and managed false, asserts the definition is skipped and no managed run + is submitted. +- Recognize the server's `incomplete_plan` terminal error (API-001). The server rejects a + plan that fails to cover every snapshotted definition with `422 incomplete_plan`; that + code is now in the SDK's `ERROR_SPECS` mirror and the shared `contract.json` fixture + (byte-identical with the server's), so a worker no longer treats a valid server-defined + failure as an unrecognized error. The fixture-equality test covers it. +- Close a heap-address disclosure bypass in the managed-source sandbox + (adversarial-audit SEC). The output-boundary guard that rejects a `` + repr in a result field was anchored on the literal `<`, so an allow-listed + `str(payload.get).replace("<", "")` — or an f-string / `%`-format of a bare bound + method — kept the live heap address while stripping the match, leaking an + ASLR/memory-layout primitive of the sandbox process into a persisted result. The fix + moves the defense to compile time: a bound method (the only reachable value with a + pointer repr — the transcript and result types are all frozen, pointer-free + dataclasses) may now only be **called**, never referenced as a bare value, so no + reachable value can carry a pointer repr through `str()`, an f-string, or `%`. The + output-boundary scan is kept and broadened (no longer requires the leading `<`) as + defense in depth. Legitimate evaluations — which call methods and read data + attributes — are unaffected; regression tests cover the `.replace("<","")`, f-string, + and `%` bypasses and confirm called-method/data-attribute stringification still works. - Switch the worker from long-polling to **normal (short) polling**, matching the cadence of our other cloud surfaces. `claim` no longer sends `wait_seconds` and the server returns immediately; when a claim comes back empty the worker sleeps diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py index ab02c09a..7383e94a 100644 --- a/sdk/python/failproofai_sdk/evaluator/protocol.py +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -61,6 +61,7 @@ "transcript_too_large": {"http_status": 413, "retryable": False}, "invalid_request": {"http_status": 422, "retryable": False}, "invalid_catalog": {"http_status": 422, "retryable": False}, + "incomplete_plan": {"http_status": 422, "retryable": False}, "unsupported_protocol_version": {"http_status": 426, "retryable": False}, "internal_error": {"http_status": 500, "retryable": True}, } diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 01abe9c5..041d4442 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -335,22 +335,28 @@ async def process_assignment(self, assignment: Assignment) -> None: self._increment("conditions_skipped") continue try: + # Whose condition decides applicability follows the execution mode, + # mirroring the evaluator branch below (`run.execution_mode`): a LOCAL + # definition's condition is client-authored (`local.condition`); a + # PYTHON (managed) definition's is server-authored and MUST govern even + # when the worker also registered the same key/version locally. Keying + # `local` on `(eval_key, eval_version)` alone means a managed def can + # collide with a local one; selecting `local.condition` there would let + # a matching local condition override the server's managed rule and run + # the managed evaluator against the operator's intent (COR-001). # Compile INSIDE the try: a managed condition the sandbox rejects # (unsafe/malformed source) must dead-letter as `condition_error`, # not raise out of the plan loop and strand the whole assignment # until its retry budget is exhausted. - condition_function = ( - local.condition - if local is not None - else ( - compile_condition( - descriptor.condition_source, - timeout_seconds=descriptor.timeout_seconds, - ) - if descriptor.condition_source - else None + if descriptor.execution_mode is ExecutionMode.LOCAL: + condition_function = local.condition if local is not None else None + elif descriptor.condition_source: + condition_function = compile_condition( + descriptor.condition_source, + timeout_seconds=descriptor.timeout_seconds, ) - ) + else: + condition_function = None if condition_function is None: selected.append((descriptor, local)) continue diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 69e24bef..976c35fb 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -323,12 +323,13 @@ def _run_sandboxed( # of pure string/collection data methods. Anything else — every current and # future introspection attribute — is rejected. `format`/`format_map` are simply # absent from this set, so the C-level format escape is closed too. -_ALLOWED_ATTRS = frozenset( +# Data attributes on the transcript surface — safe to READ as a value: each is a +# field of a frozen dataclass (SessionTranscript / TranscriptEvent, whose reprs are +# field-based and pointer-free) or a JSON scalar/container from an event payload. +_DATA_ATTRS = frozenset( { # SessionTranscript + TranscriptEvent data surface (see protocol.py). "events", - "events_of_type", - "count", "event_count", "event_type", "payload", @@ -342,6 +343,21 @@ def _run_sandboxed( "started_at", "ended_at", "schema_version", + } +) + +# Method attributes — pure data methods that must be CALLED, never referenced as a +# bare value. A bound method's repr is `<... at 0x...>`, a live heap address; a bare +# reference (`payload.get` uncalled) is only ever useful for smuggling that address +# into a result field via `str()`, an f-string, or `%`-formatting — none of which a +# real evaluation needs. `_compile` requires each of these names to appear at a call +# site, which closes every text-coercion leak at its source: no reachable value can +# then carry a pointer repr, so the output-boundary scan is only defense in depth. +_METHOD_ATTRS = frozenset( + { + # SessionTranscript methods. + "events_of_type", + "count", # dict data methods. "get", "keys", @@ -398,6 +414,10 @@ def _run_sandboxed( } ) +# The walk rejects any attribute outside this union, and additionally requires every +# name in `_METHOD_ATTRS` to appear only as the function of a call. +_ALLOWED_ATTRS = _DATA_ATTRS | _METHOD_ATTRS + def _fresh_globals() -> dict[str, Any]: """A throwaway globals mapping for one eval call. @@ -430,6 +450,14 @@ def _compile(source: str, *, field_name: str, maximum: int) -> Any: tree = ast.parse(source, mode="eval") except SyntaxError as error: raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + # An Attribute that is the function of a Call is a method invocation; any other + # Attribute naming a method (`_METHOD_ATTRS`) is a bare bound-method reference, + # whose only use is leaking the method's `<... at 0xADDR>` repr into a result. + called_method_nodes = { + node.func + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } node_count = 0 for node in ast.walk(tree): node_count += 1 @@ -466,21 +494,26 @@ def _compile(source: str, *, field_name: str, maximum: int) -> Any: raise UnsafeEvaluatorSource( f"{field_name} may not access attribute '{node.attr}'" ) + if node.attr in _METHOD_ATTRS and node not in called_method_nodes: + raise UnsafeEvaluatorSource( + f"{field_name} may reference method '{node.attr}' only to call it; " + "a bare bound method leaks a heap address when stringified" + ) if isinstance(node, ast.Name) and node.id.startswith("_"): raise UnsafeEvaluatorSource(f"{field_name} may not access private names") return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) -# CPython's default object repr — `<... at 0x7f...>` — embeds a live heap -# address (an ASLR/memory-layout disclosure). An expression cannot be stopped -# from producing such a repr at the source level: it falls out of `str()` on any -# bound method of an allowed object (`str(payload.get)`), and those methods must -# stay reachable. So the disclosure is closed at the OUTPUT boundary instead: a -# result whose text embeds this signature is rejected. The result types are all -# frozen dataclasses with pointer-free reprs, so scanning the value's repr sees -# every user-controlled string field. The pattern is the interpreter's own repr -# grammar, which authored reasoning/summaries never legitimately contain. -_OBJECT_REPR = re.compile(r"<[^<>]* at 0x[0-9a-fA-F]+") +# CPython's default object repr — `<... at 0x7f...>` — embeds a live heap address +# (an ASLR/memory-layout disclosure). The PRIMARY defense is at compile time: a bound +# method (the only reachable object with such a repr — the result and transcript types +# are all frozen, pointer-free dataclasses) can no longer be referenced as a value +# (`_METHOD_ATTRS` must be called), so no reachable value carries a pointer repr to +# begin with. This output-boundary scan is DEFENSE IN DEPTH. It matches the "... at +# 0xADDR" tail every default repr shares, plus a bare `0x`+hex run — deliberately +# WITHOUT the leading `<`, so reshaping the wrapper (e.g. `str(x).replace("<","")`, +# the way the compile-time hole was originally bypassed) cannot strip the match. +_OBJECT_REPR = re.compile(r" at 0x[0-9a-fA-F]+|0x[0-9a-fA-F]{6,}") def _forbid_object_reprs(field_name: str, value: Any) -> Any: diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json index afd09a07..afe0d4ea 100644 --- a/sdk/python/tests/fixtures/evaluator_v2/contract.json +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -54,6 +54,7 @@ "transcript_too_large": {"http_status": 413, "retryable": false}, "invalid_request": {"http_status": 422, "retryable": false}, "invalid_catalog": {"http_status": 422, "retryable": false}, + "incomplete_plan": {"http_status": 422, "retryable": false}, "unsupported_protocol_version": {"http_status": 426, "retryable": false}, "internal_error": {"http_status": 500, "retryable": true} }, diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index 8c637d9e..7f547476 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -231,6 +231,59 @@ def plan(self, assignment_id, request): assert result.summary is None +def test_managed_condition_governs_even_when_a_local_key_collides(): + # COR-001: `local` is keyed on (eval_key, eval_version) alone, so a managed + # (PYTHON) definition can collide with a local one the worker also registered. + # The server's managed condition must decide applicability — NOT the matching + # local condition. Here the local condition returns True and the managed + # `condition_source` is "False": the definition must be recorded as skipped + # (condition_false) and the managed evaluator source must never run. + source = "EvalResult(score=Score(1.0), summary='should never run')" + + evaluator = Evaluator(name="managed", version="1") + + @evaluator.eval("hosted_quality", version="1", when=lambda session: True) + def hosted_quality(session): # a colliding LOCAL definition, condition True + return EvalResult(score=Score(1.0, passed=True), summary="local") + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + condition_source="False", + source_checksum=source_checksum("False", source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + # The server's managed condition (False) wins over the local one (True): + # recorded as skipped, nothing selected, and no managed run submitted. + assert client.plans[0].selected == () + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("hosted_quality", "condition_false"), + } + assert client.submissions == [] + + def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): evaluator = Evaluator(name="parallel", version="1") lock = threading.Lock() diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 897603dc..46d98794 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -141,25 +141,65 @@ def test_no_reachable_construct_leaks_a_heap_pointer_repr(): @pytest.mark.parametrize( "inner", [ - # A bound method's repr leaks the underlying object's heap pointer, and - # these methods are allowlisted (needed by real evals) so they cannot be - # removed. The OUTPUT guard rejects the disclosure wherever it rides out. + # A bound method's repr is `<... at 0xADDR>` — a live heap pointer. These + # methods stay allowlisted (real evals CALL them), but referencing one as a + # bare VALUE only serves to stringify that repr, so it is now rejected at + # COMPILE — the disclosure is closed at its source, not at the output. "session.events[0].payload.get", "''.join", "'x'.encode", "'a,b'.split", ], ) -def test_object_repr_pointer_disclosure_is_rejected_at_the_output(inner): +def test_bare_bound_method_reference_is_rejected_at_compile(inner): for field in ( f'EvalResult(score=Score(1.0), reasoning=str({inner}))', f'EvalResult(score=Score(1.0), summary=str({inner}))', f'EvalResult(score=Score(1.0, display_value=str({inner})))', f'EvalResult(score=Score(1.0), labels=(str({inner}),))', ): - evaluate = compile_evaluator(field) - with pytest.raises(UnsafeEvaluatorSource, match="object repr"): - evaluate(Session()) + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(field) + + +def test_heap_pointer_output_guard_bypasses_are_closed_at_compile(): + # An adversarial-review finding: the output-boundary regex was anchored on `<`, + # so a managed source could keep the address while reshaping the wrapper text — + # str(...).replace("<",""), an f-string, or %-formatting all coerce a bound + # method at a point the old scan missed. Each needs a BARE bound-method + # reference, which the compile-time call-site rule now rejects outright. + bypasses = [ + # str(...).replace("<","") strips the old regex's `<` anchor. + 'EvalResult(score=Score(1.0), reasoning=str(dict().get).replace("<", ""))', + # f-strings coerce at the C level, past the `str` global. + 'EvalResult(score=Score(1.0), reasoning=f"{session.events[0].payload.get}")', + # %-formatting coerces at the C level too. + 'EvalResult(score=Score(1.0), reasoning="%s" % session.events[0].payload.get)', + ] + for src in bypasses: + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(src) + + +def test_called_methods_and_data_attributes_still_stringify(): + # The call-site rule blocks only BARE method references. Calling methods and + # reading data attributes (both pointer-free) must still work — including the + # f-string and %-formatting paths — so legitimate evaluations are unaffected. + reasoning_call = compile_evaluator( + 'EvalResult(score=Score(1.0), ' + 'reasoning=str(session.events[0].payload.get("tool_name")))' + )(Session()) + assert reasoning_call.reasoning == "search" + + fstring = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning=f"n={session.event_count}")' + )(Session()) + assert fstring.reasoning == "n=3" + + percent = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning="pct=%d" % (session.event_count * 10))' + )(Session()) + assert percent.reasoning == "pct=30" def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next(): From 34610c35f22a0980d9382f0fa4628744944463e6 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Mon, 31 Aug 2026 18:50:02 +0530 Subject: [PATCH 13/20] Count sandbox-slot wait against the execution timeout (hermes SEC-001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_run_sandboxed` acquired the MAX_CONCURRENT_SANDBOXES slot with an unbounded wait and only started its wall-clock deadline afterward. The runtime runs this in a thread and `asyncio.wait_for` cancels only the awaiter, so a run queued behind busy slots could — after its caller was already reported timed out — still acquire a slot and launch a sandbox; 28 threads could pile up behind 4 long sandboxes and starve the worker (conditions have no runtime-level wait at all). One wall-clock deadline now covers BOTH the slot wait and execution: the slot is acquired with the remaining budget, and on timeout (or a slot acquired exactly at the deadline) the run raises EvaluationTimeout without spawning a child. Regression test: more concurrent compute bombs than slots all resolve within ~one budget, not N serialized budgets. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 11 +++++ .../failproofai_sdk/evaluator/source.py | 26 +++++++++--- sdk/python/tests/test_evaluator_source.py | 42 +++++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 421afd53..34a75422 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -61,6 +61,17 @@ it ships. - Require `execution_mode` on the wire instead of coercing a falsy/missing value to `local` — a malformed value silently ran a `python` definition down the customer path (or vice-versa); it is now a hard protocol error. +- Count the sandbox-slot wait against the execution timeout (SEC-001). `_run_sandboxed` + acquired the `MAX_CONCURRENT_SANDBOXES` slot with an UNBOUNDED wait and only started + its wall-clock deadline afterward — so a run queued behind busy slots could, after the + runtime's `asyncio.wait_for` already reported it timed out (that cancels only the + awaiter, not the executor thread), still acquire a slot and launch a sandbox; 28 + threads could pile up behind 4 long sandboxes and starve the worker (conditions have + no runtime-level wait at all). One wall-clock deadline now covers BOTH the slot wait + and execution: the slot is acquired with the remaining budget, and on timeout the run + raises `EvaluationTimeout` **without spawning a child**. Regression test: more + concurrent compute bombs than slots all resolve within ~one budget, not N serialized + budgets. - A managed (`python`) definition's applicability is now governed by the SERVER's `condition_source`, never a colliding local condition (COR-001). `process_assignment` keyed the local-definition lookup on `(eval_key, eval_version)` alone and selected diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 976c35fb..f5c0c767 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -156,10 +156,25 @@ def _run_sandboxed( total = 0 timed_out = False too_large = False - # Hold a slot for the whole subprocess lifetime so no more than - # MAX_CONCURRENT_SANDBOXES run at once — bounds the aggregate memory the - # sandboxes can consume regardless of the worker's claim concurrency. - with _SANDBOX_SLOTS: + # A slot is held for the whole subprocess lifetime so no more than + # MAX_CONCURRENT_SANDBOXES run at once — bounding aggregate memory across + # concurrent sandboxes. But acquiring it must COUNT AGAINST the wall-clock + # budget: the runtime runs this in a thread and `asyncio.wait_for` only + # cancels the awaiter, so a thread that blocked here UNBOUNDED past its + # deadline would still go on to launch a sandbox after its run was already + # reported timed out — 28 such threads could queue behind 4 long sandboxes + # and starve the worker (conditions have no runtime-level wait at all). One + # deadline therefore covers BOTH the slot wait and execution: we acquire the + # slot with the remaining budget and, on failure, time out WITHOUT spawning. + deadline = time.monotonic() + wall_timeout + acquire_timeout = deadline - time.monotonic() + if acquire_timeout <= 0 or not _SANDBOX_SLOTS.acquire(timeout=acquire_timeout): + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") + try: + if deadline - time.monotonic() <= 0: + # Slot acquired exactly at the deadline: a child launched now could + # only be killed immediately, so do not spawn one at all. + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") try: proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], @@ -171,7 +186,6 @@ def _run_sandboxed( raise EvaluationSandboxUnavailable( f"could not start the evaluation sandbox: {error}" ) from error - deadline = time.monotonic() + wall_timeout out_fd = proc.stdout.fileno() try: while True: @@ -196,6 +210,8 @@ def _run_sandboxed( if proc.poll() is None: proc.kill() proc.wait() + finally: + _SANDBOX_SLOTS.release() finally: try: os.unlink(path) diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 46d98794..739cb32e 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -331,6 +331,48 @@ def test_oversized_result_is_rejected_before_it_crosses_back(): compile_evaluator(src, eval_key="q")(Session()) +def test_sandbox_slot_wait_counts_against_the_timeout(monkeypatch): + # SEC-001: acquiring a concurrency slot must count against the wall-clock budget. + # `asyncio.wait_for` only cancels the awaiter, so a run that blocked UNBOUNDED on + # a busy slot would still launch a sandbox after its caller was reported timed + # out — 28 threads could queue behind 4 long sandboxes and starve the worker. + # With one slot and three 1s compute bombs, all three must resolve within ~one + # budget (the holder is killed at ~1s; the two queued behind it exhaust their + # budget waiting and time out WITHOUT ever spawning a child), not three serialized + # budgets (~3s). + import threading + import time as _time + + from failproofai_sdk.evaluator import source as _source + + monkeypatch.setattr(_source, "_SANDBOX_SLOTS", threading.Semaphore(1)) + bomb = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + session = Session() + errors: list[str] = [] + + def run(): + try: + bomb(session) + except Exception as error: # noqa: BLE001 + errors.append(type(error).__name__) + + threads = [threading.Thread(target=run) for _ in range(3)] + started = _time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + elapsed = _time.monotonic() - started + + assert errors == ["EvaluationTimeout"] * 3, errors + # Bounded by ~one budget, NOT three serialized ones — proving the queued runs + # timed out on slot acquisition instead of each waiting then running in turn. + assert elapsed < 2.5, f"queued sandboxes were not bounded by the timeout: {elapsed:.2f}s" + + def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit(): # A ~1.6 GiB allocation exceeds the per-sandbox RLIMIT_AS and is killed, so it # cannot exhaust the worker even wrapped in an otherwise-valid result. With the From caf63b6dbeaee13f776dc0b8e8402bc1cf81316b Mon Sep 17 00:00:00 2001 From: Siddartha Aralakuppe Yogesha Date: Tue, 1 Sep 2026 11:51:44 +0530 Subject: [PATCH 14/20] fix(evaluator): report why source was rejected; scrub the sandbox child's env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found reviewing failproofai#758 alongside agenteye#652 as one system, with a real server, a managed worker and two customer-hosted workers running. Surface the reason a server-authored definition was rejected Every failure collapsed to "evaluation raised ", so a hosted definition that can never run reported only "evaluation raised UnsafeEvaluatorSource" — on every session, forever, with nothing telling the author what was wrong. It matters because the server accepts any source that passes its size and key checks: it does not (and in Rust cannot cheaply) validate the sandbox's single-expression grammar, so a definition that is structurally unrunnable is published with 201 Created and then fails silently per-session. Observed exactly that end to end: a perfectly ordinary multi-statement evaluator was accepted by the API and failed every session with no diagnosis. UnsafeEvaluatorSource now reports its detail ("evaluator_source must be one expression"), bounded to MAX_ERROR_MESSAGE_BYTES. Deliberately narrower than the generic handler, which still reports the type name only: UnsafeEvaluatorSource is raised by our own validator BEFORE any customer source executes and its message describes the source's shape, so it carries no transcript content — whereas an arbitrary eval exception can quote the transcript it was reading into a field that is persisted and displayed. Ordered before `except Exception` so it is reachable (it subclasses ValueError). Scrub the sandbox child's environment subprocess.Popen inherited os.environ, so the sandbox executing untrusted server-authored source ran with FAILPROOFAI_EVALUATOR_TOKEN in its environment — on the FailproofAI-managed pod, the cross-tenant credential the whole fleet authenticates with. The AST allowlist and empty __builtins__ stop a managed expression from reaching os.environ today, so this is defence in depth rather than a live escape: it means a future gap in those restrictions cannot be escalated into credential theft. Only what the interpreter needs is forwarded, PYTHONPATH included — without it the child cannot import the sandbox runner at all. Verified: 148 evaluator tests pass. The sandbox itself held under direct attack — open()/eval()/globals() die as NameError on empty builtins, dunder and introspection attributes are refused at compile time, and a 10**9-element allocation bomb was contained as a per-eval MemoryError in 497ms with no host memory movement. Co-Authored-By: Claude Opus 5 (1M context) --- .../failproofai_sdk/evaluator/runtime.py | 35 +++++++++++++++++++ .../failproofai_sdk/evaluator/source.py | 32 +++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index 041d4442..c0d629ae 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -26,6 +26,7 @@ from failproofai_sdk.evaluator.protocol import ( DEFAULT_POLL_INTERVAL_SECONDS, MAX_CLAIM_CAPACITY, + MAX_ERROR_MESSAGE_BYTES, MAX_WORKER_ID_BYTES, Assignment, AssignmentDefinition, @@ -42,6 +43,7 @@ ) from failproofai_sdk.evaluator.source import ( EvaluationTimeout, + UnsafeEvaluatorSource, compile_condition, compile_evaluator, source_checksum, @@ -528,7 +530,40 @@ async def _execute_run_in_slot( except asyncio.CancelledError: await self._cancel_hook(definition, session) raise + except UnsafeEvaluatorSource as error: + # Surface the REASON for a rejected server-authored definition. + # + # This is deliberately narrower than the generic handler below. + # UnsafeEvaluatorSource is raised by our own validator before any + # customer source executes, and its message is SDK-authored text + # about the source's shape ("evaluator_source must be one + # expression", "contains disallowed syntax: Assign") — it embeds no + # transcript content, so it is safe to send back over the wire. + # + # Without this the author saw only "evaluation raised + # UnsafeEvaluatorSource" on every session, with no way to learn what + # was wrong: the server accepts any source that passes its size and + # key checks, so a definition that can never run is published + # successfully and then fails silently and permanently. + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + detail = str(error).strip() + error_message = ( + f"evaluator source rejected: {detail}" + if detail + else "evaluator source rejected by the sandbox validator" + ) + encoded = error_message.encode("utf-8") + if len(encoded) > MAX_ERROR_MESSAGE_BYTES: + error_message = encoded[:MAX_ERROR_MESSAGE_BYTES].decode( + "utf-8", "ignore" + ) except Exception as error: # noqa: BLE001 - converts customer eval failures + # Type name ONLY. A customer eval's exception text can quote the + # transcript it was reading, and this field is persisted and shown + # in the dashboard, so the message itself is not repeated here. items = () status = TerminalRunStatus.FAILED summary = None diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index f5c0c767..27e284ce 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -74,6 +74,37 @@ # items, bounded fields) is far under this. SANDBOX_MAX_RESULT_BYTES = 1 * 1024 * 1024 # 1 MiB +# The sandbox child is scrubbed of the worker's environment. +# +# `subprocess.Popen` inherits `os.environ` by default, which on a worker means +# FAILPROOFAI_EVALUATOR_TOKEN — and on the FailproofAI-managed pod that token is +# the CROSS-TENANT credential the whole fleet authenticates with. The AST and +# empty-builtins restrictions already stop a managed expression from reading +# `os.environ`, so this is defence in depth rather than a fix for a live escape: +# it means a future gap in those restrictions cannot be escalated into credential +# theft. Only the variables the interpreter itself needs are forwarded — notably +# PYTHONPATH, without which the child cannot import the sandbox runner at all. +_SANDBOX_ENV_PASSTHROUGH = ( + "PATH", + "PYTHONPATH", + "PYTHONHOME", + "PYTHONDONTWRITEBYTECODE", + "PYTHONUNBUFFERED", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "SYSTEMROOT", # Windows: CPython fails to start without it +) + + +def _sandbox_env() -> dict[str, str]: + return { + name: os.environ[name] + for name in _SANDBOX_ENV_PASSTHROUGH + if os.environ.get(name) + } + def _clamp_budget(timeout_seconds: float | None) -> float: """The wall-clock/CPU budget for one evaluation: a positive value no larger @@ -181,6 +212,7 @@ def _run_sandboxed( stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + env=_sandbox_env(), ) except OSError as error: raise EvaluationSandboxUnavailable( From fedcc75030779dbd6a10da6e10423ea92adbdc8a Mon Sep 17 00:00:00 2001 From: Siddartha Aralakuppe Yogesha Date: Tue, 1 Sep 2026 11:51:44 +0530 Subject: [PATCH 15/20] test(evaluator): give the condition compute bomb a margin that survives a fast interpreter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_condition_compute_bomb_is_also_bounded` failed on CI under Python 3.14 after the previous commit, while passing locally — the signature of a machine-speed coin flip rather than a real regression. The cause is the bomb's size, not the sandbox. The test asserts that a CPU-bound condition cannot finish inside its budget, but used `sum(range(10**8))` against a 1-second budget: ~1.35 CPU-seconds measured on 3.14, a 1.35x margin. On a fast enough runner the sum simply completes and nothing times out. 3.14 is the version that fails first because it is the fastest — 1.35s against 3.13's 1.44s here. Its evaluator twin one function above already uses `10**9` (~13x margin) for the same 1-second budget, so the condition variant was carrying a bomb ten times smaller for no stated reason. Matching it restores the margin and costs no wall-clock: the sandbox kills the child at its budget either way, so a bigger bomb only widens the gap between "killed" and "could have finished". The test still completes in ~1.06s. This changes a test rather than the code because the code is correct — the property under test (a CPU bomb in a condition is stopped by the sandbox budget) is unchanged and now actually verified rather than raced. The threshold was the defect. Also records this PR's two SDK fixes in the changelog section they belong to. Verified with CI's own command on the version that failed: `uv sync --locked --extra dev --python 3.14 && uv run pytest tests/ -q` — 961 passed, 9 skipped, 0 failed. Co-Authored-By: Claude Opus 5 (1M context) --- sdk/python/CHANGELOG.md | 20 ++++++++++++++++++++ sdk/python/tests/test_evaluator_source.py | 17 ++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 34a75422..0d2cb85e 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -31,6 +31,26 @@ it ships. any bound method's repr) is rejected at the output boundary. `enumerate` and bare generator expressions are no longer permitted — both were gratuitous pointer-repr sources; use `range(len(...))` and list/set/dict comprehensions. +- Report **why** a server-authored definition was rejected. Every failure + collapsed to `evaluation raised `, so a hosted definition that can + never run reported only `evaluation raised UnsafeEvaluatorSource` — on every + session, forever, with nothing telling the author what was wrong. It matters + because the server accepts any source passing its size and key checks and does + not validate the sandbox's single-expression grammar, so a structurally + unrunnable definition is published successfully and then fails silently. + `UnsafeEvaluatorSource` now carries its detail (`evaluator_source must be one + expression`), bounded to `MAX_ERROR_MESSAGE_BYTES`. Deliberately narrower than + the generic handler, which still reports the type name only: this exception is + raised by our own validator before any customer source executes and describes + the source's shape, so it embeds no transcript content. +- Scrub the sandbox child's environment. `subprocess.Popen` inherited + `os.environ`, so the process executing untrusted server-authored source ran + with `FAILPROOFAI_EVALUATOR_TOKEN` in its environment — on the managed pod, + the cross-tenant credential. Defence in depth rather than a live escape (the + AST allowlist and empty `__builtins__` already stop a managed expression + reaching `os.environ`): a future gap there can no longer be escalated into + credential theft. Only what the interpreter needs is forwarded, `PYTHONPATH` + included. - Contain a poison managed definition to its own run: source is now compiled lazily inside the per-run executor, so a definition the sandbox rejects dead-letters as one bounded `failed`/`eval_error` run instead of crashing the diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index 739cb32e..a666b538 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -269,7 +269,22 @@ def test_compute_bomb_is_killed_within_its_budget(): def test_condition_compute_bomb_is_also_bounded(): - condition = compile_condition("sum(range(10**8)) > 0", timeout_seconds=1) + # `10**9`, matching the evaluator bomb above, NOT `10**8`. + # + # The property under test is "a CPU bomb in a condition is stopped by the + # sandbox budget", and the bomb has to be big enough that it cannot finish + # inside that budget on ANY machine the suite runs on. At 10**8 it was only + # ~1.35 CPU-seconds against a 1-second budget — a 1.35x margin — so on a fast + # runner the sum simply completed and nothing timed out. It failed exactly + # that way on CI under Python 3.14, which is faster here than 3.13 (1.35s vs + # 1.44s measured), while passing locally: a machine-speed coin flip, not a + # real signal about the sandbox. + # + # 10**9 restores the ~13x margin the evaluator twin already had. It costs no + # extra wall-clock: the sandbox kills the child at its budget either way, so + # a bigger bomb only widens the gap between "killed" and "could have + # finished". Do not shrink it back. + condition = compile_condition("sum(range(10**9)) > 0", timeout_seconds=1) with pytest.raises(EvaluationTimeout): condition(Session()) From bdf3a8f32f469fa9bded19b229f718a6045b5312 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Tue, 1 Sep 2026 17:37:17 +0530 Subject: [PATCH 16/20] fp-cloud-cli: mirror the new evaluations:run permission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `evaluations:run` to ALL_PERMISSIONS (in the server's declared order, after evaluations:trigger). It is a normal key-assignable grant — the credential a customer evaluator pod authenticates with — so `fp keys create --permission evaluations:run` and `fp users` accept it, it lands in the admin preset, and it stays out of read-only/standard. Mirrors server auth.rs. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- fp-cloud-cli/fp_cli/permissions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fp-cloud-cli/fp_cli/permissions.py b/fp-cloud-cli/fp_cli/permissions.py index 39560990..e475074c 100644 --- a/fp-cloud-cli/fp_cli/permissions.py +++ b/fp-cloud-cli/fp_cli/permissions.py @@ -26,6 +26,7 @@ "users:delete", "evaluations:read", "evaluations:trigger", + "evaluations:run", "dashboards:read", "dashboards:write", "dashboards:delete", From 4f5a9a428465527ef711cc5a1bb2b5dbdb930fd0 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Tue, 1 Sep 2026 20:04:43 +0530 Subject: [PATCH 17/20] sdk(evaluator): fail closed when kernel resource limits are unavailable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEC-001 (hermes review): on a platform without the stdlib `resource` module (e.g. Windows), the sandbox child's `_install_limits` no-ops, so a permitted expression could allocate unbounded memory in each managed-worker child before the parent's wall-clock kill lands — the advertised RLIMIT_AS cap is never imposed. `_run_sandboxed` now refuses with `EvaluationSandboxUnavailable` BEFORE spawning any child when `_resource` is unavailable, rather than run server-authored source unbounded. Regression test verifies no child is started. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/failproofai_sdk/evaluator/source.py | 10 ++++++++++ sdk/python/tests/test_evaluator_source.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 27e284ce..2bc7f4e4 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -167,6 +167,16 @@ def _run_sandboxed( nor an oversized result (``metrics={str(x):1 for x in range(100000)}``) can exhaust the worker. """ + # SEC-001: without the stdlib ``resource`` module (e.g. Windows) the sandbox + # child cannot install RLIMIT_CPU / RLIMIT_AS on itself (``_install_limits`` + # no-ops), so a permitted expression could allocate unbounded memory before + # the parent's wall-clock kill lands. Refuse BEFORE spawning any child rather + # than run server-authored source without the advertised limits. + if _resource is None: # pragma: no cover - non-POSIX + raise EvaluationSandboxUnavailable( + "kernel resource limits (RLIMIT_CPU/RLIMIT_AS) are unavailable on this " + "platform; managed evaluation cannot be bounded, refusing to run" + ) try: session_wire = session.to_wire() except AttributeError as error: diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index a666b538..b31201f3 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -395,3 +395,20 @@ def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit(): src = "EvalResult(score=Score(1.0 if len([0] * 200000000) >= 0 else 0.0))" with pytest.raises((EvaluationTimeout, MemoryError)): compile_evaluator(src, timeout_seconds=5)(Session()) + + +def test_sandbox_fails_closed_when_kernel_resource_limits_are_unavailable(monkeypatch): + # SEC-001: on a platform without the stdlib ``resource`` module (e.g. Windows), + # the sandbox child cannot install RLIMIT_CPU / RLIMIT_AS on itself, so managed + # source must be refused BEFORE any child is spawned rather than run unbounded. + import failproofai_sdk.evaluator.source as source + + monkeypatch.setattr(source, "_resource", None) + + def _no_spawn(*args, **kwargs): + raise AssertionError("a sandbox child must not be started when limits are unavailable") + + monkeypatch.setattr(source.subprocess, "Popen", _no_spawn) + + with pytest.raises(EvaluationSandboxUnavailable): + source.compile_evaluator("EvalResult(score=Score(1.0))")(Session()) From da51e5960b624d40334b69a33acdde9792b3bcad Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Tue, 1 Sep 2026 21:21:35 +0530 Subject: [PATCH 18/20] sdk(evaluator): bound the condition phase by the lease; make sync-eval thread leaks observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hermes advisories on the v2 worker runtime. Pre-plan condition phase vs the lease. Conditions were evaluated serially with no lease awareness before the plan was created, and a managed condition may run its full sandbox budget — so a few near-budget conditions could burn the whole assignment lease before the plan request, and the server then fences the plan as lease_lost and reclaims the assignment in a loop instead of recording a result. The lease cannot be renewed before the plan (the server only extends a lease for a planned assignment with running runs), so the worker now bounds the phase to the lease: each condition is capped to the time remaining before a plan-submission margin (using assignment.lease_expires_at when it is in the future, else the negotiated lease duration measured from now, which keeps the bound from firing spuriously on a replayed/stale deadline in tests or under clock skew), and once that budget is gone the remaining conditions are skipped as `lease_exhausted` rather than run. Local conditions, which previously ran with no timeout at all, are bounded the same way. The complete fix — renewing the lease during the condition phase — needs a server-side pre-plan heartbeat and is tracked separately. Timed-out synchronous evaluators. A synchronous evaluator that overruns its timeout runs in the executor thread and cannot be cancelled (CPython cannot interrupt a running thread), so its thread is orphaned; with the pool sized to the concurrency limit, one orphan on a single-slot worker silently stopped all further local evaluation. The eval executor now carries headroom over the semaphore so an orphaned thread does not immediately starve live capacity — the semaphore stays the true concurrency bound — and each orphan increments `sync_evaluations_orphaned` and logs a warning naming the evaluator so a hung one is findable. This is a finite cushion, not a cure for a permanently-blocked evaluator; the CHANGELOG points authors at async or managed evaluators for long or untrusted work. Tests: sync-timeout orphan counting + executor headroom; lease-exhausted conditions are skipped without running; lease-phase deadline honors a future lease and falls back to the duration for a stale one; condition budget caps by both remaining lease and per-definition timeout. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 24 ++++ .../failproofai_sdk/evaluator/runtime.py | 119 ++++++++++++++++-- sdk/python/tests/test_evaluator_runtime.py | 85 +++++++++++++ 3 files changed, 220 insertions(+), 8 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 0d2cb85e..100244fd 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -131,6 +131,30 @@ it ships. 25 s. Removes the `claim_wait_seconds` config knob and the `request_timeout_seconds > claim_wait_seconds` constraint; the poll cadence is now tuned centrally by the server, not per worker. +- Bound the pre-plan condition phase by the assignment lease (hermes advisory). + Conditions were evaluated serially with no lease awareness before the plan was + created, and a managed condition could run its full sandbox budget — so a few + near-budget conditions could burn the whole lease before the plan request and + the server would fence the plan as `lease_lost`, reclaiming the assignment in a + loop instead of submitting a result. Each condition is now capped to the lease + time remaining before a plan-submission margin (using `lease_expires_at` when it + is in the future, else the negotiated lease duration), and once that budget is + gone the remaining conditions are skipped as `lease_exhausted` rather than run. + Local conditions, which previously had no timeout at all, are bounded the same + way. The complete fix — renewing the lease *during* the condition phase — needs + a server-side pre-plan heartbeat and is tracked separately. +- Make a timed-out **synchronous** evaluator observable and stop it starving the + worker (hermes advisory). A synchronous evaluator that overruns its timeout runs + in the executor thread and cannot be cancelled (CPython cannot interrupt a + running thread), so its thread was permanently lost; with the pool sized to the + concurrency limit, one such orphan on a single-slot worker silently stopped all + further local evaluation. The eval executor now carries headroom over the + semaphore so an orphaned thread does not immediately starve live capacity — the + semaphore stays the real concurrency bound — and each orphan increments + `sync_evaluations_orphaned` and logs a warning naming the evaluator, so a hung + one is findable. This is a finite cushion, not a cure for a permanently-blocked + evaluator; prefer `async def` evaluators (cooperatively cancellable) or managed + `python` evaluators (subprocess-isolated, hard-killed) for long or untrusted work. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py index c0d629ae..0dba852c 100644 --- a/sdk/python/failproofai_sdk/evaluator/runtime.py +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -51,6 +51,25 @@ logger = logging.getLogger("failproofai_sdk.evaluator") +# A synchronous evaluator (or condition) that overruns its timeout cannot be +# cancelled: the executor thread runs the customer function to completion no +# matter what `asyncio.wait_for` does, because CPython cannot interrupt a running +# thread. To stop one such orphaned thread from starving live capacity, the eval +# executor is sized with headroom OVER the concurrency limit — the semaphore, not +# the thread pool, stays the real bound on how many evaluations run at once. This +# is a finite cushion, not a cure: a permanently-blocked synchronous evaluator +# invoked once per session leaks one thread per session, and no fixed pool +# survives that. `sync_evaluations_orphaned` and a warning make the offending +# evaluator findable; prefer `async def` evaluators (cooperatively cancellable) or +# managed PYTHON evaluators (subprocess-isolated, hard-killed) for long or +# untrusted work. +_EVAL_EXECUTOR_ORPHAN_HEADROOM = 8 + +# Reserved out of the lease for the plan request (and network jitter) so the +# pre-plan condition phase always leaves time to submit the plan before the lease +# expires. See `WorkerRuntime._condition_phase_deadline`. +_CONDITION_PHASE_SAFETY_MARGIN_SECONDS = 5.0 + def _utc_now() -> str: return ( @@ -180,8 +199,12 @@ def __init__( self._lease_duration = 120 self._disabled_definitions: set[str] = set() self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + # Headroom over the semaphore so a timed-out-but-still-running synchronous + # evaluator (an unkillable orphaned thread) does not immediately starve + # live capacity — the semaphore remains the true concurrency bound. See + # `_EVAL_EXECUTOR_ORPHAN_HEADROOM`. self._eval_executor = concurrent.futures.ThreadPoolExecutor( - max_workers=config.max_concurrency, + max_workers=config.max_concurrency + _EVAL_EXECUTOR_ORPHAN_HEADROOM, thread_name_prefix="failproof-eval", ) self._registered = False @@ -326,6 +349,12 @@ async def process_assignment(self, assignment: Assignment) -> None: (item.eval_key, item.eval_version): item for item in self.evaluator.definitions } + # The assignment lease is fixed at claim time and cannot be renewed until + # the plan is submitted (the server only extends a lease for a *planned* + # assignment with running runs). A slow condition phase can therefore burn + # the whole lease and get the plan fenced as lease_lost, so every + # condition is bounded by the lease it must leave time to plan within. + condition_deadline = self._condition_phase_deadline(assignment) for descriptor in descriptors: local = local_definitions.get( (descriptor.eval_key, descriptor.eval_version) @@ -350,19 +379,42 @@ async def process_assignment(self, assignment: Assignment) -> None: # (unsafe/malformed source) must dead-letter as `condition_error`, # not raise out of the plan loop and strand the whole assignment # until its retry budget is exhausted. + managed_condition_source: str | None = None if descriptor.execution_mode is ExecutionMode.LOCAL: condition_function = local.condition if local is not None else None elif descriptor.condition_source: - condition_function = compile_condition( - descriptor.condition_source, - timeout_seconds=descriptor.timeout_seconds, - ) + # Compiled below, once the lease budget is known, so the + # sandbox subprocess is bounded by whatever lease remains. + managed_condition_source = descriptor.condition_source + condition_function = None else: condition_function = None - if condition_function is None: + if condition_function is None and managed_condition_source is None: + # No condition to run — applicable by default, no lease spent. selected.append((descriptor, local)) continue - condition = await self._invoke(condition_function, session) + budget = self._condition_budget( + condition_deadline, descriptor.timeout_seconds + ) + if budget <= 0.0: + # Not enough lease left to evaluate this condition and still + # submit the plan in time; skip it (and, as the loop proceeds, + # every later condition) rather than do work the server will + # fence as lease_lost and reclaim in a loop. + skipped.append( + self._skipped_descriptor(descriptor, "lease_exhausted") + ) + self._increment("conditions_skipped") + self._increment("conditions_lease_exhausted") + continue + if managed_condition_source is not None: + condition_function = compile_condition( + managed_condition_source, + timeout_seconds=budget, + ) + condition = await asyncio.wait_for( + self._invoke(condition_function, session), timeout=budget + ) if isinstance(condition, ConditionResult): applicable = condition.applicable reason_code = condition.reason_code @@ -495,6 +547,9 @@ async def _execute_run_in_slot( ) -> None: started_at = _utc_now() started = time.monotonic() + # A synchronous evaluator runs in the executor thread; if it overruns the + # wall-clock timeout below, the thread cannot be cancelled and is orphaned. + sync_function = not inspect.iscoroutinefunction(definition.function) try: invocation = self._invoke(definition.function, session) result = ( @@ -517,11 +572,27 @@ async def _execute_run_in_slot( summary = result.summary error_code = None error_message = None - except (asyncio.TimeoutError, EvaluationTimeout): + except (asyncio.TimeoutError, EvaluationTimeout) as timeout_error: # asyncio.TimeoutError: the awaiter hit the wall-clock. EvaluationTimeout: # the forked managed sandbox was killed by its CPU/memory/time budget — # the real, thread-uncancellable case. Both are a timed-out run. await self._cancel_hook(definition, session) + if isinstance(timeout_error, asyncio.TimeoutError) and sync_function: + # The awaiter gave up while a SYNCHRONOUS evaluator was still + # running in the executor. CPython cannot interrupt that thread, + # so it is now orphaned — it runs to completion (or forever) + # holding a worker thread. Count it and name the evaluator so a + # hung one is findable; the executor's headroom keeps this one + # orphan from immediately starving live capacity. + self._increment("sync_evaluations_orphaned") + logger.warning( + "synchronous evaluation exceeded its timeout and cannot be " + "cancelled; its worker thread is orphaned until it returns", + extra={ + "assignment_id": assignment.assignment_id, + "eval_key": definition.eval_key, + }, + ) items = () status = TerminalRunStatus.TIMED_OUT summary = None @@ -648,6 +719,38 @@ async def _heartbeat( ) self._increment("heartbeat_failures") + def _condition_phase_deadline(self, assignment: Assignment) -> float: + """Monotonic-clock reading by which the pre-plan condition phase must end. + + The real `lease_expires_at` is used when it is in the future (production); + a past or unparseable value (clock skew, or a replayed transcript in a + test) falls back to the negotiated lease duration measured from now, so + the bound never fires spuriously on a stale deadline. + """ + remaining = float(self._lease_duration) + try: + expires = datetime.fromisoformat( + assignment.lease_expires_at.replace("Z", "+00:00") + ) + parsed = (expires - datetime.now(timezone.utc)).total_seconds() + if parsed > 0: + remaining = parsed + except (ValueError, AttributeError): + pass + return time.monotonic() + remaining + + def _condition_budget( + self, deadline: float, timeout_seconds: int | None + ) -> float: + """Seconds a single condition may run: the lease left before the + plan-submission margin, capped by the definition's own timeout.""" + remaining = ( + deadline - time.monotonic() - _CONDITION_PHASE_SAFETY_MARGIN_SECONDS + ) + if timeout_seconds is not None: + remaining = min(remaining, float(timeout_seconds)) + return remaining + async def _invoke(self, function, session): if inspect.iscoroutinefunction(function): return await function(session) diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py index 7f547476..dff10869 100644 --- a/sdk/python/tests/test_evaluator_runtime.py +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -1016,3 +1016,88 @@ async def measured(session): assert peak == 1 DefinitionsResponse, ExecutionMode, + + +def test_synchronous_evaluation_timeout_is_counted_as_orphaned(): + # A synchronous evaluator that overruns its timeout cannot be cancelled: the + # runtime submits a terminal timed_out result and records the orphaned thread + # so a hung evaluator is findable. The executor is sized with headroom over + # the concurrency limit so this orphan does not starve live capacity. + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1", timeout_seconds=0.05) + def slow(session): + time.sleep(0.5) + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + try: + asyncio.run(runtime.process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert runtime.metrics().get("sync_evaluations_orphaned") == 1 + assert runtime._eval_executor._max_workers > runtime.config.max_concurrency + finally: + runtime._eval_executor.shutdown(wait=True) + + +def test_conditions_are_skipped_when_the_lease_is_exhausted(): + # With no lease time left before the plan must be submitted, the worker skips + # the condition (without running it) instead of burning the lease and getting + # the plan fenced as lease_lost. + evaluator = Evaluator(name="test", version="1") + ran = [] + + def gate(session): + ran.append(True) + return True + + @evaluator.eval("slow", version="1", when=gate) + def slow(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + # Force an already-exhausted condition-phase deadline. + runtime._condition_phase_deadline = lambda assignment: time.monotonic() + asyncio.run(runtime.process_assignment(client.assignment)) + + assert ran == [], "the condition must not run once the lease is exhausted" + assert client.plans, "a plan must still be submitted" + plan_request = client.plans[-1] + assert not plan_request.selected + reasons = {(s.eval_key, s.reason_code) for s in plan_request.skipped} + assert ("slow", "lease_exhausted") in reasons + assert runtime.metrics().get("conditions_lease_exhausted") == 1 + assert client.submissions == [] + + +def test_condition_phase_deadline_and_budget_are_lease_bounded(): + from datetime import datetime, timedelta, timezone + + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = _runtime(evaluator, client) + runtime._lease_duration = 120 + + # A stale (past) lease_expires_at falls back to the negotiated lease duration, + # so the bound never fires spuriously under clock skew or a replayed fixture. + stale = replace(client.assignment, lease_expires_at="2000-01-01T00:00:00.000000Z") + fallback = runtime._condition_phase_deadline(stale) - time.monotonic() + assert 110 <= fallback <= 125 + + # A future lease is honored. + future_ts = (datetime.now(timezone.utc) + timedelta(seconds=300)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + "Z" + future = replace(client.assignment, lease_expires_at=future_ts) + ahead = runtime._condition_phase_deadline(future) - time.monotonic() + assert 250 <= ahead <= 305 + + # Budget is capped by both the remaining lease and the per-definition timeout. + deadline = time.monotonic() + 100 + assert runtime._condition_budget(deadline, None) == pytest.approx(95, abs=2) + assert runtime._condition_budget(deadline, 10) == pytest.approx(10, abs=0.05) + assert runtime._condition_budget(time.monotonic(), None) < 0 From 16a85a82371e35a56645afe2c5ecd25b877f80d6 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Tue, 1 Sep 2026 22:19:14 +0530 Subject: [PATCH 19/20] sdk(evaluator): let a managed comprehension read session on Python 3.10 (hermes COR-001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sandbox eval passed `session` in the eval locals mapping (eval(code, fresh_globals, {"session": session})). A list/set/dict comprehension runs in its own scope and resolves a free name like `session` from GLOBALS, so on CPython 3.10/3.11 — supported versions — an allowed source such as `all([session.event_count > 0 for i in range(1)])` raised NameError. `session` now goes in the fresh per-call globals mapping and both the condition and evaluator eval paths use empty locals, so a comprehension resolves it while per-call isolation is preserved. Verified on 3.10 (reproduced the NameError without the fix) and passing 3.10–3.14. Regression test: test_comprehension_body_can_read_session. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- sdk/python/CHANGELOG.md | 8 ++++++++ sdk/python/failproofai_sdk/evaluator/source.py | 16 ++++++++++++++-- sdk/python/tests/test_evaluator_source.py | 16 ++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 100244fd..e06eb6cc 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -155,6 +155,14 @@ it ships. one is findable. This is a finite cushion, not a cure for a permanently-blocked evaluator; prefer `async def` evaluators (cooperatively cancellable) or managed `python` evaluators (subprocess-isolated, hard-killed) for long or untrusted work. +- Let a managed source's list/set/dict comprehension read `session` on CPython + 3.10/3.11 (hermes COR-001). The sandbox eval put `session` in the eval *locals*, + but a comprehension runs in its own scope and resolves a free name like + `session` from *globals* — so on 3.10 (a supported version) an allowed source + such as `all([session.event_count > 0 for i in range(1)])` raised `NameError`. + `session` now goes in a fresh per-call globals mapping and both eval paths use + empty locals, which keeps isolation and works across 3.10–3.14. Regression test + runs on the whole version matrix. ## 0.0.1b1 — 2026-08-24 diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py index 2bc7f4e4..8327c7f8 100644 --- a/sdk/python/failproofai_sdk/evaluator/source.py +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -597,7 +597,13 @@ def _raw_eval(source: str, kind: str) -> Callable[[Any], Any]: ) def run(session: Any) -> Any: - value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 if not isinstance(value, (bool, ConditionResult)): raise TypeError("condition_source must return bool or ConditionResult") return _forbid_object_reprs("condition_source", value) @@ -608,7 +614,13 @@ def run(session: Any) -> Any: ) def run(session: Any) -> Any: - value = eval(code, _fresh_globals(), {"session": session}) # noqa: S307 + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 if not isinstance(value, EvalResult): raise TypeError("evaluator_source must return EvalResult") return _forbid_object_reprs("evaluator_source", value) diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py index b31201f3..fc65bcd5 100644 --- a/sdk/python/tests/test_evaluator_source.py +++ b/sdk/python/tests/test_evaluator_source.py @@ -412,3 +412,19 @@ def _no_spawn(*args, **kwargs): with pytest.raises(EvaluationSandboxUnavailable): source.compile_evaluator("EvalResult(score=Score(1.0))")(Session()) + + +def test_comprehension_body_can_read_session(): + # A list/set/dict comprehension resolves a free name like `session` from + # GLOBALS. When `session` was only in eval locals, such a source raised + # NameError on CPython 3.10 (a supported version). Both sandbox paths must + # now evaluate a session-dependent comprehension (COR-001). + assert ( + compile_condition("len([session.event_count for i in range(1)]) > 0")(Session()) + is True + ) + result = compile_evaluator( + "EvalResult(score=Score(1.0), " + "reasoning=str([session.event_count for i in range(1)]))" + )(Session(event_count=3)) + assert result.reasoning == "[3]" From 8cfdb598a4477b3ee6691d3a73fa05651227d276 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Tue, 1 Sep 2026 22:42:14 +0530 Subject: [PATCH 20/20] deps: bump browserslist to 4.28.8 to clear GHSA-73wf-gq98-2v4g (OSV gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Supply Chain OSV-Scanner gate started failing on a newly-published High advisory (GHSA-73wf-gq98-2v4g, 7.5) against browserslist 4.28.2, which sits in bun.lock transitively via @babel/helper-compilation-targets (range ^4.24.0). main only passes because its last scan predates the advisory; every fresh scan, including this PR's, now blocks on it. The finding is fixable (4.28.7+), and osv-scanner.toml says to prefer fixing over ignoring, so this pins browserslist to 4.28.8 through the existing package.json `overrides` block (the repo's established pin mechanism, alongside undici/sharp/…) rather than adding an IgnoredVulns entry. The lockfile change is contained to browserslist and its own data deps (caniuse-lite, electron-to-chromium, node-releases, update-browserslist-db, baseline-browser-mapping); no other package moves, and `bun install --frozen-lockfile` is clean. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_018v4UWTsTpQaxMGdt7sJVtW --- bun.lock | 13 +++++++++---- package.json | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bun.lock b/bun.lock index 3312d169..b74bdc96 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ }, "overrides": { "brace-expansion": "5.0.9", + "browserslist": "4.28.8", "eslint-plugin-react-hooks": "7.0.1", "nanoid": "3.3.18", "postcss": "8.5.26", @@ -493,7 +494,7 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="], + "browserslist": ["browserslist@4.28.8", "", { "dependencies": { "baseline-browser-mapping": "^2.11.12", "caniuse-lite": "^1.0.30001809", "electron-to-chromium": "^1.5.402", "node-releases": "^2.0.53", "update-browserslist-db": "^1.3.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA=="], "call-bind": ["call-bind@1.0.9", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "get-intrinsic": "^1.3.0", "set-function-length": "^1.2.2" } }, "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ=="], @@ -567,7 +568,7 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - "electron-to-chromium": ["electron-to-chromium@1.5.363", "", {}, "sha512-VjUKPyWzGnT1fujlkEGC/BvN70Hh70KXtAqcmniXviYlJC/ivcT+BWGPyxWVbJZLfvtKR6dqg1L7T7pgAMBtWA=="], + "electron-to-chromium": ["electron-to-chromium@1.5.418", "", {}, "sha512-UzS26r3AEbG5wSoGVpJKqwHIU9zwQN7LHdVIThDrJpS0I5KdlXFMEb8543fhc9dVnIIAST6ar8rhwa00AL5MlA=="], "emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], @@ -981,7 +982,7 @@ "node-exports-info": ["node-exports-info@1.6.0", "", { "dependencies": { "array.prototype.flatmap": "^1.3.3", "es-errors": "^1.3.0", "object.entries": "^1.1.9", "semver": "^6.3.1" } }, "sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw=="], - "node-releases": ["node-releases@2.0.46", "", {}, "sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ=="], + "node-releases": ["node-releases@2.0.54", "", {}, "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ=="], "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], @@ -1235,7 +1236,7 @@ "unrs-resolver": ["unrs-resolver@1.12.2", "", { "dependencies": { "napi-postinstall": "^0.3.4" }, "optionalDependencies": { "@unrs/resolver-binding-android-arm-eabi": "1.12.2", "@unrs/resolver-binding-android-arm64": "1.12.2", "@unrs/resolver-binding-darwin-arm64": "1.12.2", "@unrs/resolver-binding-darwin-x64": "1.12.2", "@unrs/resolver-binding-freebsd-x64": "1.12.2", "@unrs/resolver-binding-linux-arm-gnueabihf": "1.12.2", "@unrs/resolver-binding-linux-arm-musleabihf": "1.12.2", "@unrs/resolver-binding-linux-arm64-gnu": "1.12.2", "@unrs/resolver-binding-linux-arm64-musl": "1.12.2", "@unrs/resolver-binding-linux-loong64-gnu": "1.12.2", "@unrs/resolver-binding-linux-loong64-musl": "1.12.2", "@unrs/resolver-binding-linux-ppc64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-musl": "1.12.2", "@unrs/resolver-binding-linux-s390x-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-musl": "1.12.2", "@unrs/resolver-binding-openharmony-arm64": "1.12.2", "@unrs/resolver-binding-wasm32-wasi": "1.12.2", "@unrs/resolver-binding-win32-arm64-msvc": "1.12.2", "@unrs/resolver-binding-win32-ia32-msvc": "1.12.2", "@unrs/resolver-binding-win32-x64-msvc": "1.12.2" } }, "sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ=="], - "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], + "update-browserslist-db": ["update-browserslist-db@1.3.2", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw=="], "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], @@ -1317,6 +1318,10 @@ "@typescript-eslint/typescript-estree/tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], + "browserslist/baseline-browser-mapping": ["baseline-browser-mapping@2.11.20", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw=="], + + "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001810", "", {}, "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg=="], + "data-urls/whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "^1.11.0", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="], "eslint-import-resolver-node/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], diff --git a/package.json b/package.json index f5b4d75a..ac481e27 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "vite": "8.0.16", "undici": "7.29.0", "brace-expansion": "5.0.9", - "sharp": "0.35.0" + "sharp": "0.35.0", + "browserslist": "4.28.8" } }