diff --git a/bun.lock b/bun.lock index 3312d169..b74bdc96 100644 --- a/bun.lock +++ b/bun.lock @@ -39,6 +39,7 @@ }, "overrides": { "brace-expansion": "5.0.9", + "browserslist": "4.28.8", "eslint-plugin-react-hooks": "7.0.1", "nanoid": "3.3.18", "postcss": "8.5.26", @@ -493,7 +494,7 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="], + "browserslist": ["browserslist@4.28.8", "", { "dependencies": { "baseline-browser-mapping": "^2.11.12", "caniuse-lite": "^1.0.30001809", "electron-to-chromium": "^1.5.402", "node-releases": "^2.0.53", "update-browserslist-db": "^1.3.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA=="], "call-bind": ["call-bind@1.0.9", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "get-intrinsic": "^1.3.0", "set-function-length": "^1.2.2" } }, "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ=="], @@ -567,7 +568,7 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], - "electron-to-chromium": ["electron-to-chromium@1.5.363", "", {}, "sha512-VjUKPyWzGnT1fujlkEGC/BvN70Hh70KXtAqcmniXviYlJC/ivcT+BWGPyxWVbJZLfvtKR6dqg1L7T7pgAMBtWA=="], + "electron-to-chromium": ["electron-to-chromium@1.5.418", "", {}, "sha512-UzS26r3AEbG5wSoGVpJKqwHIU9zwQN7LHdVIThDrJpS0I5KdlXFMEb8543fhc9dVnIIAST6ar8rhwa00AL5MlA=="], "emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], @@ -981,7 +982,7 @@ "node-exports-info": ["node-exports-info@1.6.0", "", { "dependencies": { "array.prototype.flatmap": "^1.3.3", "es-errors": "^1.3.0", "object.entries": "^1.1.9", "semver": "^6.3.1" } }, "sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw=="], - "node-releases": ["node-releases@2.0.46", "", {}, "sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ=="], + "node-releases": ["node-releases@2.0.54", "", {}, "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ=="], "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], @@ -1235,7 +1236,7 @@ "unrs-resolver": ["unrs-resolver@1.12.2", "", { "dependencies": { "napi-postinstall": "^0.3.4" }, "optionalDependencies": { "@unrs/resolver-binding-android-arm-eabi": "1.12.2", "@unrs/resolver-binding-android-arm64": "1.12.2", "@unrs/resolver-binding-darwin-arm64": "1.12.2", "@unrs/resolver-binding-darwin-x64": "1.12.2", "@unrs/resolver-binding-freebsd-x64": "1.12.2", "@unrs/resolver-binding-linux-arm-gnueabihf": "1.12.2", "@unrs/resolver-binding-linux-arm-musleabihf": "1.12.2", "@unrs/resolver-binding-linux-arm64-gnu": "1.12.2", "@unrs/resolver-binding-linux-arm64-musl": "1.12.2", "@unrs/resolver-binding-linux-loong64-gnu": "1.12.2", "@unrs/resolver-binding-linux-loong64-musl": "1.12.2", "@unrs/resolver-binding-linux-ppc64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-gnu": "1.12.2", "@unrs/resolver-binding-linux-riscv64-musl": "1.12.2", "@unrs/resolver-binding-linux-s390x-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-gnu": "1.12.2", "@unrs/resolver-binding-linux-x64-musl": "1.12.2", "@unrs/resolver-binding-openharmony-arm64": "1.12.2", "@unrs/resolver-binding-wasm32-wasi": "1.12.2", "@unrs/resolver-binding-win32-arm64-msvc": "1.12.2", "@unrs/resolver-binding-win32-ia32-msvc": "1.12.2", "@unrs/resolver-binding-win32-x64-msvc": "1.12.2" } }, "sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ=="], - "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], + "update-browserslist-db": ["update-browserslist-db@1.3.2", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw=="], "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], @@ -1317,6 +1318,10 @@ "@typescript-eslint/typescript-estree/tinyglobby": ["tinyglobby@0.2.16", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg=="], + "browserslist/baseline-browser-mapping": ["baseline-browser-mapping@2.11.20", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw=="], + + "browserslist/caniuse-lite": ["caniuse-lite@1.0.30001810", "", {}, "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg=="], + "data-urls/whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "^1.11.0", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="], "eslint-import-resolver-node/debug": ["debug@3.2.7", "", { "dependencies": { "ms": "^2.1.1" } }, "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ=="], diff --git a/fp-cloud-cli/fp_cli/permissions.py b/fp-cloud-cli/fp_cli/permissions.py index 39560990..e475074c 100644 --- a/fp-cloud-cli/fp_cli/permissions.py +++ b/fp-cloud-cli/fp_cli/permissions.py @@ -26,6 +26,7 @@ "users:delete", "evaluations:read", "evaluations:trigger", + "evaluations:run", "dashboards:read", "dashboards:write", "dashboards:delete", diff --git a/package.json b/package.json index f5b4d75a..ac481e27 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ "vite": "8.0.16", "undici": "7.29.0", "brace-expansion": "5.0.9", - "sharp": "0.35.0" + "sharp": "0.35.0", + "browserslist": "4.28.8" } } diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 57aa24a9..e06eb6cc 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,6 +16,154 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. +- Retire the old inbound evaluator boundary and add evaluator authoring plus the + outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator` + namespace. +- Harden the managed-evaluator source sandbox against a class of escapes an + adversarial review found: `str.format`/`format_map` C-level field traversal, + generator/frame introspection (`gi_frame.f_globals`) that reached the eval + globals and could poison a process-shared namespace across evaluations, and + `type.mro()` type-object reach. Attribute access is now **default-deny** (an + allowlist of the transcript data surface plus pure string/collection methods, + so every current and future introspection attribute is rejected), each eval + runs with **fresh per-call globals**, and a result whose text embeds a runtime + object repr (`<... at 0x...>`, a heap-pointer/ASLR disclosure that falls out of + any bound method's repr) is rejected at the output boundary. `enumerate` and + bare generator expressions are no longer permitted — both were gratuitous + pointer-repr sources; use `range(len(...))` and list/set/dict comprehensions. +- Report **why** a server-authored definition was rejected. Every failure + collapsed to `evaluation raised `, so a hosted definition that can + never run reported only `evaluation raised UnsafeEvaluatorSource` — on every + session, forever, with nothing telling the author what was wrong. It matters + because the server accepts any source passing its size and key checks and does + not validate the sandbox's single-expression grammar, so a structurally + unrunnable definition is published successfully and then fails silently. + `UnsafeEvaluatorSource` now carries its detail (`evaluator_source must be one + expression`), bounded to `MAX_ERROR_MESSAGE_BYTES`. Deliberately narrower than + the generic handler, which still reports the type name only: this exception is + raised by our own validator before any customer source executes and describes + the source's shape, so it embeds no transcript content. +- Scrub the sandbox child's environment. `subprocess.Popen` inherited + `os.environ`, so the process executing untrusted server-authored source ran + with `FAILPROOFAI_EVALUATOR_TOKEN` in its environment — on the managed pod, + the cross-tenant credential. Defence in depth rather than a live escape (the + AST allowlist and empty `__builtins__` already stop a managed expression + reaching `os.environ`): a future gap there can no longer be escalated into + credential theft. Only what the interpreter needs is forwarded, `PYTHONPATH` + included. +- Contain a poison managed definition to its own run: source is now compiled + lazily inside the per-run executor, so a definition the sandbox rejects + dead-letters as one bounded `failed`/`eval_error` run instead of crashing the + assignment task and forcing it to be reclaimed until its attempt budget runs + out. +- Run managed (server-authored) evaluations in a **killable fork+exec'd + subprocess** with hard `RLIMIT_CPU` + `RLIMIT_AS` + a parent wall-clock kill — + so a compute/memory bomb in a hosted definition (`sum(range(10**20))`) can no + longer exhaust the worker (SEC-001). Cancelling an in-process thread does not + stop it; a fresh subprocess the kernel bounds and the parent terminates does. A + plain `os.fork()` would deadlock — the worker is multi-threaded (asyncio loop, + executor, writer) and forking one hangs the child on an inherited lock — so the + sandbox execs a fresh `python -m ..._sandbox_runner` that sets its own limits; + the transcript crosses in via `to_wire`, only the result crosses back. The + effective budget is **clamped to a hard ceiling** (`MAX_SANDBOX_TIMEOUT_SECONDS`, + 60s) so a large server-provided `timeout_seconds` cannot remove the bound. + Managed conditions, which previously ran with no timeout at all, are sandboxed + the same way. The result crossing back is **bounded on both sides** — the child + validates it (`result_items`, the 25-result limit) and refuses to serialize + anything over 1 MiB, and the parent reads at most that before killing the child + — so an oversized result (`metrics={str(x): 1 for x in range(100000)}`) cannot + OOM the worker either. The per-sandbox address space is capped (512 MiB) and the number of concurrent sandbox processes is bounded (a semaphore), so the AGGREGATE memory is bounded independent of the worker's `max_concurrency` — a fleet of concurrent runs can't OOM the host. Fails **closed** (`EvaluationSandboxUnavailable`) if the + sandbox cannot be spawned or the transcript cannot be serialized. Defense in depth at + compile time: reject `**` with a large/non-constant exponent and cap total AST + size. A managed condition the sandbox rejects now dead-letters as + `condition_error` instead of stranding the assignment. Only server-authored + source is isolated this way; customer evaluators still run in-process. +- Require `execution_mode` on the wire instead of coercing a falsy/missing value + to `local` — a malformed value silently ran a `python` definition down the + customer path (or vice-versa); it is now a hard protocol error. +- Count the sandbox-slot wait against the execution timeout (SEC-001). `_run_sandboxed` + acquired the `MAX_CONCURRENT_SANDBOXES` slot with an UNBOUNDED wait and only started + its wall-clock deadline afterward — so a run queued behind busy slots could, after the + runtime's `asyncio.wait_for` already reported it timed out (that cancels only the + awaiter, not the executor thread), still acquire a slot and launch a sandbox; 28 + threads could pile up behind 4 long sandboxes and starve the worker (conditions have + no runtime-level wait at all). One wall-clock deadline now covers BOTH the slot wait + and execution: the slot is acquired with the remaining budget, and on timeout the run + raises `EvaluationTimeout` **without spawning a child**. Regression test: more + concurrent compute bombs than slots all resolve within ~one budget, not N serialized + budgets. +- A managed (`python`) definition's applicability is now governed by the SERVER's + `condition_source`, never a colliding local condition (COR-001). `process_assignment` + keyed the local-definition lookup on `(eval_key, eval_version)` alone and selected + `local.condition` whenever a local definition with that key existed — so a managed + definition whose server condition was false could be forced to run anyway if the + worker had also registered a local definition under the same key whose condition was + true, executing server-managed source against the operator's intent. Condition + selection now branches on `execution_mode`, mirroring the evaluator branch: `LOCAL` + uses `local.condition`, `PYTHON` compiles and runs the server's `condition_source` + regardless of any key collision. Regression test: identical local+managed keys, local + condition true and managed false, asserts the definition is skipped and no managed run + is submitted. +- Recognize the server's `incomplete_plan` terminal error (API-001). The server rejects a + plan that fails to cover every snapshotted definition with `422 incomplete_plan`; that + code is now in the SDK's `ERROR_SPECS` mirror and the shared `contract.json` fixture + (byte-identical with the server's), so a worker no longer treats a valid server-defined + failure as an unrecognized error. The fixture-equality test covers it. +- Close a heap-address disclosure bypass in the managed-source sandbox + (adversarial-audit SEC). The output-boundary guard that rejects a `` + repr in a result field was anchored on the literal `<`, so an allow-listed + `str(payload.get).replace("<", "")` — or an f-string / `%`-format of a bare bound + method — kept the live heap address while stripping the match, leaking an + ASLR/memory-layout primitive of the sandbox process into a persisted result. The fix + moves the defense to compile time: a bound method (the only reachable value with a + pointer repr — the transcript and result types are all frozen, pointer-free + dataclasses) may now only be **called**, never referenced as a bare value, so no + reachable value can carry a pointer repr through `str()`, an f-string, or `%`. The + output-boundary scan is kept and broadened (no longer requires the leading `<`) as + defense in depth. Legitimate evaluations — which call methods and read data + attributes — are unaffected; regression tests cover the `.replace("<","")`, f-string, + and `%` bypasses and confirm called-method/data-attribute stringification still works. +- Switch the worker from long-polling to **normal (short) polling**, matching the + cadence of our other cloud surfaces. `claim` no longer sends `wait_seconds` and + the server returns immediately; when a claim comes back empty the worker sleeps + the server-advertised `poll_interval_seconds` (from the register response, + default 10 s) before polling again, instead of holding a request open for up to + 25 s. Removes the `claim_wait_seconds` config knob and the + `request_timeout_seconds > claim_wait_seconds` constraint; the poll cadence is + now tuned centrally by the server, not per worker. +- Bound the pre-plan condition phase by the assignment lease (hermes advisory). + Conditions were evaluated serially with no lease awareness before the plan was + created, and a managed condition could run its full sandbox budget — so a few + near-budget conditions could burn the whole lease before the plan request and + the server would fence the plan as `lease_lost`, reclaiming the assignment in a + loop instead of submitting a result. Each condition is now capped to the lease + time remaining before a plan-submission margin (using `lease_expires_at` when it + is in the future, else the negotiated lease duration), and once that budget is + gone the remaining conditions are skipped as `lease_exhausted` rather than run. + Local conditions, which previously had no timeout at all, are bounded the same + way. The complete fix — renewing the lease *during* the condition phase — needs + a server-side pre-plan heartbeat and is tracked separately. +- Make a timed-out **synchronous** evaluator observable and stop it starving the + worker (hermes advisory). A synchronous evaluator that overruns its timeout runs + in the executor thread and cannot be cancelled (CPython cannot interrupt a + running thread), so its thread was permanently lost; with the pool sized to the + concurrency limit, one such orphan on a single-slot worker silently stopped all + further local evaluation. The eval executor now carries headroom over the + semaphore so an orphaned thread does not immediately starve live capacity — the + semaphore stays the real concurrency bound — and each orphan increments + `sync_evaluations_orphaned` and logs a warning naming the evaluator, so a hung + one is findable. This is a finite cushion, not a cure for a permanently-blocked + evaluator; prefer `async def` evaluators (cooperatively cancellable) or managed + `python` evaluators (subprocess-isolated, hard-killed) for long or untrusted work. +- Let a managed source's list/set/dict comprehension read `session` on CPython + 3.10/3.11 (hermes COR-001). The sandbox eval put `session` in the eval *locals*, + but a comprehension runs in its own scope and resolves a free name like + `session` from *globals* — so on 3.10 (a supported version) an allowed source + such as `all([session.event_count > 0 for i in range(1)])` raised `NameError`. + `session` now goes in a fresh per-call globals mapping and both eval paths use + empty locals, which keeps isolation and works across 3.10–3.14. Regression test + runs on the whole version matrix. + ## 0.0.1b1 — 2026-08-24 The first release under this name. Everything below describes the package as it diff --git a/sdk/python/README.md b/sdk/python/README.md index f974587d..3cb8adaf 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -10,6 +10,14 @@ the platform. - **Dependencies:** none. Standard library only, so installing it constrains nothing else in your environment. +## Evaluator v2 status + +The legacy inbound `agenteye-evaluator` package has been retired; do not build new +evaluator services against its server-push HTTP contract. Evaluator v2 authoring +and its customer-hosted, outbound-only worker runtime live under the lazy +`failproofai_sdk.evaluator` namespace. Importing the top-level tracing SDK does not +import or start the evaluator runtime. + ## Installation ```bash diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py new file mode 100644 index 00000000..912b61e5 --- /dev/null +++ b/sdk/python/examples/evaluator_worker.py @@ -0,0 +1,121 @@ +"""Customer evaluator with deterministic and optional async judge checks.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import json +import os +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator import ( + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, +) + +app = Evaluator(name="customer-production", version="2026.08.1") + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +@app.eval( + "tool_efficiency", + version="1.0.0", + labels=["tools", "deterministic"], + when=lambda session: ConditionResult( + session.count("tool_use") > 0, "no_tool_calls" + ), +) +def tool_efficiency(session): + calls = session.events_of_type("tool_use") + distinct = { + event.payload.get("tool_name") + for event in calls + if event.payload.get("tool_name") + } + value = len(distinct) / len(calls) + return EvalResult( + score=Score(value, passed=value >= 0.7), + metrics={ + "tool_call_count": Metric(len(calls), unit="events"), + "distinct_tool_count": Metric(len(distinct), unit="tools"), + }, + reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls", + ) + + +def _judge_configured(session): + configured = bool(os.environ.get("EXAMPLE_JUDGE_URL")) + return ConditionResult(configured, "judge_not_configured") + + +def _last_content(session, event_type): + events = session.events_of_type(event_type) + if not events: + return None + payload = events[-1].payload + fields = { + "human_input": ("response",), + "model_response": ("content",), + "agent_end": ("summary",), + }.get(event_type, ("content", "summary", "response")) + return next((payload.get(field) for field in fields if payload.get(field)), None) + + +def _call_judge(question, answer): + url = os.environ["EXAMPLE_JUDGE_URL"] + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback: + raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback") + token = os.environ.get("EXAMPLE_JUDGE_TOKEN") + body = json.dumps({"question": question, "answer": answer}).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + request = Request(url, data=body, headers=headers, method="POST") + with build_opener(_RejectRedirects()).open(request, timeout=25) as response: # nosec B310 + result = json.loads(response.read(64 * 1024)) + return float(result["score"]), str( + result.get("reasoning") or "Judge returned no reasoning" + ) + + +@app.eval( + "answer_relevance", + version="judge-api-v1", + labels=["llm_judge", "relevance"], + when=_judge_configured, + timeout_seconds=30, +) +async def answer_relevance(session): + question = _last_content(session, "human_input") + answer = _last_content(session, "model_response") + if question is None or answer is None: + raise ValueError("answer relevance requires human input and model output") + value, reasoning = await asyncio.to_thread(_call_judge, question, answer) + value = min(max(value, 0.0), 1.0) + return EvalResult( + score=Score(value, passed=value >= 0.7), + reasoning=reasoning, + labels=("llm_judge", "relevance"), + ) + + +if __name__ == "__main__": + app.run_from_env() diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py new file mode 100644 index 00000000..a94155ce --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -0,0 +1,101 @@ +"""Authoring and worker primitives for FailproofAI Evaluator v2. + +This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users +who only emit telemetry do not import evaluator networking or runtime code. +""" + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, + Metric, + Score, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + Assignment, + AssignmentDefinition, + CatalogDefinition, + ClaimRequest, + ClaimResponse, + ErrorResponse, + EvalSelection, + ExecutionMode, + EvaluatorKind, + HeartbeatRequest, + HeartbeatResponse, + HeartbeatRun, + PlannedRun, + PlanRequest, + PlanResponse, + DefinitionsResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + RemoteError, + ResultItem, + ResultKind, + ResultRequest, + ResultResponse, + SessionTranscript, + SkippedEval, + TerminalRunStatus, + TranscriptEvent, + UnsupportedProtocolVersion, +) +from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime +from failproofai_sdk.evaluator.source import ( + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + +__all__ = [ + "Assertion", + "Assignment", + "AssignmentDefinition", + "CatalogDefinition", + "ClaimRequest", + "ClaimResponse", + "ConditionResult", + "ErrorResponse", + "EvalDefinition", + "EvalResult", + "EvalSelection", + "ExecutionMode", + "Evaluator", + "EvaluatorAPIError", + "EvaluatorClient", + "EvaluatorKind", + "HeartbeatRequest", + "HeartbeatResponse", + "HeartbeatRun", + "Metric", + "PlanRequest", + "PlanResponse", + "DefinitionsResponse", + "PlannedRun", + "ProtocolError", + "RegisterRequest", + "RegisterResponse", + "RemoteError", + "ResultItem", + "ResultKind", + "ResultRequest", + "ResultResponse", + "Score", + "SessionTranscript", + "SkippedEval", + "TerminalRunStatus", + "TranscriptEvent", + "UnsupportedProtocolVersion", + "WorkerConfig", + "WorkerRuntime", + "UnsafeEvaluatorSource", + "compile_condition", + "compile_evaluator", + "source_checksum", +] diff --git a/sdk/python/failproofai_sdk/evaluator/__main__.py b/sdk/python/failproofai_sdk/evaluator/__main__.py new file mode 100644 index 00000000..f7bff1c5 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__main__.py @@ -0,0 +1,49 @@ +"""Run an evaluator declared as ``module:attribute``.""" + +from __future__ import annotations + +import argparse +import importlib +import os +from collections.abc import Sequence + +from failproofai_sdk.evaluator.authoring import Evaluator + + +def load_evaluator(spec: str) -> Evaluator: + module_name, separator, attribute = spec.partition(":") + if not module_name: + raise ValueError("evaluator module must not be empty") + if not separator: + attribute = "app" + if not attribute: + raise ValueError("evaluator attribute must not be empty") + module = importlib.import_module(module_name) + try: + evaluator = getattr(module, attribute) + except AttributeError as error: + raise ValueError(f"{spec!r} does not define {attribute!r}") from error + if not isinstance(evaluator, Evaluator): + raise TypeError( + f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator" + ) + return evaluator + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator") + parser.add_argument( + "module", + nargs="?", + default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"), + help="Python module and optional attribute (for example my_evals:app)", + ) + args = parser.parse_args(argv) + if not args.module: + parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)") + load_evaluator(args.module).run_from_env() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py new file mode 100644 index 00000000..1b773e26 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py @@ -0,0 +1,63 @@ +"""Subprocess entry point for the managed-source sandbox. + +Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner `` by +``source._run_sandboxed``. The input file holds a pickled +``(kind, source, session_wire, cpu_seconds, mem_bytes, eval_key)`` tuple. This +process installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates +the re-validated server-authored source against the reconstructed transcript, +validates + bounds the result, and writes a pickled ``("ok", result)`` / +``("err", type_name, message)`` outcome to stdout. + +This is a FRESH exec'd process — never a fork of the multi-threaded worker — so +there is no inherited-lock deadlock (forking a process that has an asyncio loop, +an executor pool and a writer daemon hangs the child). The parent enforces the +wall-clock bound and the output-size bound by reading only so far and killing +this process on timeout or overflow. +""" + +from __future__ import annotations + +import pickle +import sys + + +def _main() -> int: + with open(sys.argv[1], "rb") as handle: + kind, source, session_wire, cpu_seconds, mem_bytes, eval_key = pickle.loads( + handle.read() + ) + # Imported here, in the child, so the import cost is never on the worker's path. + from failproofai_sdk.evaluator.protocol import SessionTranscript + from failproofai_sdk.evaluator.source import ( + SANDBOX_MAX_RESULT_BYTES, + _install_limits, + _raw_eval, + ) + + try: + session = SessionTranscript.from_wire(session_wire) + # Compile + re-validate BEFORE the limits so validation cost is not charged + # against the eval's CPU budget; the limits bind the eval itself. + run = _raw_eval(source, kind) + _install_limits(cpu_seconds, mem_bytes) + result = run(session) + # Bound the result INSIDE the sandbox before it crosses back: result_items + # enforces the 25-result limit + field validation, so a huge result + # (`metrics={str(x):1 for x in range(100000)}`) raises here instead of + # being serialized and shipped to the parent. + if kind == "evaluator": + result.result_items(eval_key or "result") + payload = pickle.dumps(("ok", result)) + if len(payload) > SANDBOX_MAX_RESULT_BYTES: + payload = pickle.dumps( + ("err", "ResultTooLarge", "evaluation result exceeds the size limit") + ) + except BaseException as error: # noqa: BLE001 - relay type+msg, this process is the boundary + payload = pickle.dumps(("err", type(error).__name__, str(error)[:500])) + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/sdk/python/failproofai_sdk/evaluator/authoring.py b/sdk/python/failproofai_sdk/evaluator/authoring.py new file mode 100644 index 00000000..db1788d9 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/authoring.py @@ -0,0 +1,377 @@ +"""Evaluator definition registry and typed author results.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import re +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from failproofai_sdk.evaluator.protocol import ( + MAX_CATALOG_DEFINITIONS, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABEL_BYTES, + MAX_LABELS_PER_RESULT, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + CatalogDefinition, + ResultItem, + ResultKind, + SessionTranscript, +) + +_KEY = re.compile(r"^[a-z][a-z0-9_]*$") +EvalFunction = Callable[[SessionTranscript], "EvalResult | Awaitable[EvalResult]"] +ConditionFunction = Callable[ + [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]" +] +CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"] + + +def _bounded(value: str, *, field_name: str, maximum: int) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + if not value: + raise ValueError(f"{field_name} must not be empty") + size = len(value.encode("utf-8")) + if size > maximum: + raise ValueError(f"{field_name} is {size} bytes; maximum is {maximum}") + return value + + +def _finite(value: float, field_name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{field_name} must be a number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field_name} must be finite") + return result + + +def _labels(values: tuple[str, ...] | list[str]) -> tuple[str, ...]: + if len(values) > MAX_LABELS_PER_RESULT: + raise ValueError(f"at most {MAX_LABELS_PER_RESULT} labels are allowed") + normalized = [] + for label in values: + normalized.append(_bounded(label, field_name="label", maximum=MAX_LABEL_BYTES)) + if len(set(normalized)) != len(normalized): + raise ValueError("labels must be unique") + return tuple(sorted(normalized)) + + +@dataclass(frozen=True) +class Score: + value: float + passed: bool | None = None + unit: str = "ratio" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + value = _finite(self.value, "score value") + if not 0 <= value <= 1: + raise ValueError("score value must be between 0 and 1") + object.__setattr__(self, "value", value) + if self.passed is not None and not isinstance(self.passed, bool): + raise TypeError("score passed must be a boolean or None") + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Metric: + value: float + unit: str = "" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "value", _finite(self.value, "metric value")) + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Assertion: + passed: bool + description: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.passed, bool): + raise TypeError("assertion passed must be a boolean") + if self.description is not None: + _bounded( + self.description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class ConditionResult: + applicable: bool + reason_code: str = "condition_false" + + def __post_init__(self) -> None: + if not isinstance(self.applicable, bool): + raise TypeError("condition applicable must be a boolean") + _validate_key(self.reason_code, "condition reason code") + + +def _validate_result_text( + unit: str, display_value: str | None, description: str | None +) -> None: + if unit: + _bounded(unit, field_name="unit", maximum=MAX_UNIT_BYTES) + if display_value is not None: + _bounded( + display_value, + field_name="display value", + maximum=MAX_DISPLAY_VALUE_BYTES, + ) + if description is not None: + _bounded( + description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class EvalResult: + score: Score | None = None + metrics: Mapping[str, Metric | float] = field(default_factory=dict) + assertions: Mapping[str, Assertion | bool] = field(default_factory=dict) + reasoning: str | None = None + summary: str | None = None + labels: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.reasoning is not None: + _bounded( + self.reasoning, + field_name="reasoning", + maximum=MAX_REASONING_BYTES, + ) + if self.summary is not None: + _bounded(self.summary, field_name="summary", maximum=MAX_SUMMARY_BYTES) + object.__setattr__(self, "labels", _labels(list(self.labels))) + + def result_items(self, eval_key: str) -> tuple[ResultItem, ...]: + items: list[ResultItem] = [] + if self.score is not None: + items.append( + ResultItem( + result_key=eval_key, + result_kind=ResultKind.SCORE, + numeric_value=self.score.value, + bool_value=self.score.passed, + unit=self.score.unit, + display_value=self.score.display_value, + description=self.score.description, + reasoning=self.reasoning, + labels=self.labels, + ) + ) + for key, raw_metric in sorted(self.metrics.items()): + _validate_key(key, "metric key") + metric = ( + raw_metric if isinstance(raw_metric, Metric) else Metric(raw_metric) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.METRIC, + numeric_value=metric.value, + unit=metric.unit, + display_value=metric.display_value, + description=metric.description, + labels=self.labels, + ) + ) + for key, raw_assertion in sorted(self.assertions.items()): + _validate_key(key, "assertion key") + assertion = ( + raw_assertion + if isinstance(raw_assertion, Assertion) + else Assertion(raw_assertion) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.ASSERTION, + bool_value=assertion.passed, + description=assertion.description, + labels=self.labels, + ) + ) + if not items: + raise ValueError("an EvalResult must contain a score, metric, or assertion") + if len(items) > MAX_RESULTS_PER_RUN: + raise ValueError( + f"an EvalResult may contain at most {MAX_RESULTS_PER_RUN} results" + ) + keys = [item.result_key for item in items] + if len(keys) != len(set(keys)): + raise ValueError("result keys must be unique within one evaluation run") + return tuple(items) + + +def _validate_key(value: str, field_name: str = "eval_key") -> str: + _bounded(value, field_name=field_name, maximum=MAX_EVAL_KEY_BYTES) + if not _KEY.fullmatch(value): + raise ValueError(f"{field_name} must match {_KEY.pattern}") + return value + + +@dataclass(frozen=True) +class EvalDefinition: + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] + function: EvalFunction + condition: ConditionFunction | None + on_cancel: CancellationFunction | None + timeout_seconds: float | None + + def catalog_definition(self) -> CatalogDefinition: + return CatalogDefinition( + eval_key=self.eval_key, + display_name=self.display_name, + eval_version=self.eval_version, + result_kind=self.result_kind, + labels=self.labels, + ) + + +class Evaluator: + """A process-local collection of explicitly versioned evaluations.""" + + def __init__(self, *, name: str, version: str) -> None: + self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES) + self.version = _bounded( + version, field_name="version", maximum=MAX_VERSION_BYTES + ) + self._definitions: dict[str, EvalDefinition] = {} + + def eval( + self, + eval_key: str, + *, + version: str, + display_name: str | None = None, + result_kind: ResultKind | str = ResultKind.SCORE, + labels: tuple[str, ...] | list[str] = (), + when: ConditionFunction | None = None, + on_cancel: CancellationFunction | None = None, + timeout_seconds: float | None = None, + ) -> Callable[[EvalFunction], EvalFunction]: + key = _validate_key(eval_key) + eval_version = _bounded( + version, field_name="eval version", maximum=MAX_VERSION_BYTES + ) + display = _bounded( + display_name or eval_key.replace("_", " ").capitalize(), + field_name="display name", + maximum=MAX_DISPLAY_NAME_BYTES, + ) + kind = ResultKind(result_kind) + normalized_labels = _labels(list(labels)) + if timeout_seconds is not None: + timeout_seconds = _finite(timeout_seconds, "timeout_seconds") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + + def register(function: EvalFunction) -> EvalFunction: + if key in self._definitions: + raise ValueError(f"duplicate eval key: {key}") + if len(self._definitions) >= MAX_CATALOG_DEFINITIONS: + raise ValueError( + f"an evaluator may define at most {MAX_CATALOG_DEFINITIONS} evaluations" + ) + if not callable(function): + raise TypeError("evaluation must be callable") + if when is not None and not callable(when): + raise TypeError("when must be callable") + if on_cancel is not None and not callable(on_cancel): + raise TypeError("on_cancel must be callable") + self._definitions[key] = EvalDefinition( + eval_key=key, + display_name=display, + eval_version=eval_version, + result_kind=kind, + labels=normalized_labels, + function=function, + condition=when, + on_cancel=on_cancel, + timeout_seconds=timeout_seconds, + ) + return function + + return register + + @property + def definitions(self) -> tuple[EvalDefinition, ...]: + return tuple(self._definitions[key] for key in sorted(self._definitions)) + + def catalog(self) -> tuple[CatalogDefinition, ...]: + return tuple(definition.catalog_definition() for definition in self.definitions) + + @property + def catalog_revision(self) -> str: + payload = [item.to_wire() for item in self.catalog()] + canonical = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(canonical).hexdigest() + + def definition(self, eval_key: str) -> EvalDefinition: + try: + return self._definitions[eval_key] + except KeyError as error: + raise KeyError(f"unknown eval key: {eval_key}") from error + + def run_from_env(self) -> None: + """Run this evaluator until the process receives a stop request.""" + import asyncio + import signal + + from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + + async def run() -> None: + runtime = WorkerRuntime(self, WorkerConfig.from_env()) + loop = asyncio.get_running_loop() + for name in ("SIGINT", "SIGTERM"): + process_signal = getattr(signal, name, None) + if process_signal is None: + continue + try: + loop.add_signal_handler(process_signal, runtime.stop) + except (NotImplementedError, RuntimeError): + pass + await runtime.run_forever() + + asyncio.run(run()) + + @staticmethod + async def call( + function: EvalFunction | ConditionFunction, session: SessionTranscript + ) -> Any: + result = function(session) + if inspect.isawaitable(result): + return await result + return result diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py new file mode 100644 index 00000000..fc3c28ce --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -0,0 +1,299 @@ +"""Standard-library HTTP client for the Evaluator v2 worker protocol.""" + +from __future__ import annotations + +import ipaddress +import json +import random +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator.protocol import ( + CLAIM_PATH, + DEFINITIONS_PATH, + HEARTBEAT_PATH, + LEASE_GENERATION_HEADER, + MAX_TRANSCRIPT_BYTES, + PLAN_PATH, + REGISTER_PATH, + RESULT_PATH, + WORKER_ID_HEADER, + Assignment, + DefinitionsResponse, + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + WireModel, +) + +_DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024 +_RETRYABLE_HTTP_STATUSES = frozenset({429, 502, 503, 504}) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def _open_without_redirects(request: Request, *, timeout: float): + return build_opener(_RejectRedirects()).open(request, timeout=timeout) + + +class EvaluatorAPIError(RuntimeError): + def __init__( + self, + *, + status: int | None, + code: str, + message: str, + retryable: bool, + request_id: str | None = None, + ) -> None: + super().__init__(f"{code}: {message}") + self.status = status + self.code = code + self.retryable = retryable + self.request_id = request_id + + +class EvaluatorClient: + """Client for the public Evaluator v2 machine API. + + Hosted workers normally use the FailproofAI dashboard origin. Its ``/v1`` + passthrough forwards this worker's bearer credential to the private server. + """ + + def __init__( + self, + *, + base_url: str, + credential: str, + timeout_seconds: float = 30, + max_retries: int = 3, + allow_insecure_http: bool = False, + opener: Callable[..., Any] | None = None, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + parsed = urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("base_url must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback and not allow_insecure_http: + raise ValueError("base_url must use https unless it targets loopback") + if not credential or not credential.strip(): + raise ValueError("credential must not be empty") + if any( + ord(character) < 32 or ord(character) == 127 for character in credential + ): + raise ValueError("credential must not contain control characters") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if max_retries < 0: + raise ValueError("max_retries must not be negative") + self._base_url = base_url.rstrip("/") + "/" + self._origin = (parsed.scheme, parsed.netloc) + self._credential = credential + self._timeout_seconds = timeout_seconds + self._max_retries = max_retries + self._opener = opener or _open_without_redirects + self._sleeper = sleeper + + def register(self, request: RegisterRequest) -> RegisterResponse: + return RegisterResponse.from_wire( + self._json("POST", REGISTER_PATH, request, retry=True) + ) + + def claim(self, request: ClaimRequest) -> ClaimResponse: + # A lost claim response may already have leased work. Do not hide a + # second claim behind transport retry; the runtime recalculates capacity. + return ClaimResponse.from_wire( + self._json("POST", CLAIM_PATH, request, retry=False) + ) + + def transcript( + self, assignment: Assignment, *, worker_id: str + ) -> SessionTranscript: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + return SessionTranscript.from_wire( + self._json( + "GET", + assignment.transcript_url, + None, + retry=True, + headers=headers, + response_limit=MAX_TRANSCRIPT_BYTES, + ) + ) + + def definitions( + self, assignment: Assignment, *, worker_id: str + ) -> DefinitionsResponse: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + path = assignment.definitions_url or DEFINITIONS_PATH.format( + assignment_id=assignment.assignment_id + ) + return DefinitionsResponse.from_wire( + self._json("GET", path, None, retry=True, headers=headers) + ) + + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: + return PlanResponse.from_wire( + self._json( + "POST", + PLAN_PATH.format(assignment_id=assignment_id), + request, + retry=True, + ) + ) + + def heartbeat(self, request: HeartbeatRequest) -> HeartbeatResponse: + return HeartbeatResponse.from_wire( + self._json("POST", HEARTBEAT_PATH, request, retry=True) + ) + + def submit_result(self, run_id: str, request: ResultRequest) -> ResultResponse: + return ResultResponse.from_wire( + self._json( + "POST", + RESULT_PATH.format(evaluation_run_id=run_id), + request, + retry=True, + ) + ) + + def _url(self, path: str) -> str: + url = urljoin(self._base_url, path) + parsed = urlsplit(url) + if (parsed.scheme, parsed.netloc) != self._origin: + raise EvaluatorAPIError( + status=None, + code="invalid_transcript_url", + message="server supplied a URL outside the configured API origin", + retryable=False, + ) + return url + + def _json( + self, + method: str, + path: str, + body: WireModel | None, + *, + retry: bool, + headers: Mapping[str, str] | None = None, + response_limit: int = _DEFAULT_RESPONSE_LIMIT, + ) -> dict[str, Any]: + encoded = None + request_headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self._credential}", + "User-Agent": "failproofai-sdk-evaluator/2", + } + if body is not None: + encoded = json.dumps( + body.to_wire(), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + + attempts = self._max_retries + 1 if retry else 1 + for attempt in range(attempts): + request = Request( + self._url(path), data=encoded, headers=request_headers, method=method + ) + try: + with self._opener(request, timeout=self._timeout_seconds) as response: + return self._decode( + response.read(response_limit + 1), response_limit + ) + except HTTPError as error: + api_error = self._http_error(error, response_limit) + if attempt + 1 == attempts or not api_error.retryable: + raise api_error from error + except (URLError, TimeoutError, OSError) as error: + if attempt + 1 == attempts: + raise EvaluatorAPIError( + status=None, + code="transport_error", + message=str(error), + retryable=True, + ) from error + # Jitter is scheduling noise, not a security decision. + self._sleeper(random.uniform(0, min(0.25 * (2**attempt), 2.0))) # nosec B311 + raise AssertionError("retry loop exhausted without returning or raising") + + @staticmethod + def _decode(raw: bytes, limit: int) -> dict[str, Any]: + if len(raw) > limit: + raise EvaluatorAPIError( + status=None, + code="response_too_large", + message=f"server response exceeds {limit} bytes", + retryable=False, + ) + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response was not valid JSON", + retryable=False, + ) from error + if not isinstance(value, dict): + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response must be a JSON object", + retryable=False, + ) + return value + + @classmethod + def _http_error(cls, error: HTTPError, limit: int) -> EvaluatorAPIError: + raw = error.read(limit + 1) + try: + response = ErrorResponse.from_wire(cls._decode(raw, limit)) + except (ValueError, EvaluatorAPIError): + return EvaluatorAPIError( + status=error.code, + code="http_error", + message=f"server returned HTTP {error.code}", + retryable=error.code in _RETRYABLE_HTTP_STATUSES, + ) + return EvaluatorAPIError( + status=error.code, + code=response.error.code, + message=response.error.message, + retryable=response.error.retryable, + request_id=response.error.request_id, + ) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py new file mode 100644 index 00000000..7383e94a --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -0,0 +1,753 @@ +"""Dependency-free wire models for the outbound Evaluator v2 protocol.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any, TypeVar + +PROTOCOL_VERSION = "2" +TRANSCRIPT_SCHEMA_VERSION = "2" +RESULT_SCHEMA_VERSION = "2" + +REGISTER_PATH = "/v1/evaluator/workers/register" +CLAIM_PATH = "/v1/evaluator/assignments/claim" +TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +DEFINITIONS_PATH = "/v1/evaluator/assignments/{assignment_id}/definitions" +PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" +HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" +RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" +WORKER_ID_HEADER = "X-FailproofAI-Worker-Id" +LEASE_GENERATION_HEADER = "X-FailproofAI-Lease-Generation" + +HEARTBEAT_INTERVAL_SECONDS = 30 +LEASE_DURATION_SECONDS = 120 +# Fallback poll cadence if the register response omits poll_interval_seconds. The +# worker prefers the server-advertised value; claims are normal short polls, never +# long-polls, so this only bounds idle latency, not connection lifetime. +DEFAULT_POLL_INTERVAL_SECONDS = 10 +MAX_ATTEMPTS = 5 + +MAX_CATALOG_DEFINITIONS = 100 +MAX_CLAIM_CAPACITY = 32 +MAX_TRANSCRIPT_BYTES = 25 * 1024 * 1024 +MAX_RESULTS_PER_RUN = 25 +MAX_EVAL_KEY_BYTES = 128 +MAX_DISPLAY_NAME_BYTES = 128 +MAX_VERSION_BYTES = 128 +MAX_WORKER_ID_BYTES = 128 +MAX_LABEL_BYTES = 64 +MAX_LABELS_PER_RESULT = 20 +MAX_SUMMARY_BYTES = 4 * 1024 +MAX_REASONING_BYTES = 16 * 1024 +MAX_UNIT_BYTES = 64 +MAX_DISPLAY_VALUE_BYTES = 256 +MAX_DESCRIPTION_BYTES = 1_000 +MAX_ERROR_CODE_BYTES = 64 +MAX_ERROR_MESSAGE_BYTES = 4 * 1024 + +ERROR_SPECS = { + "invalid_credentials": {"http_status": 401, "retryable": False}, + "instance_disabled": {"http_status": 403, "retryable": False}, + "assignment_not_found": {"http_status": 404, "retryable": False}, + "run_not_found": {"http_status": 404, "retryable": False}, + "catalog_mismatch": {"http_status": 409, "retryable": False}, + "lease_lost": {"http_status": 409, "retryable": False}, + "plan_conflict": {"http_status": 409, "retryable": False}, + "submission_conflict": {"http_status": 409, "retryable": False}, + "retry_budget_exhausted": {"http_status": 409, "retryable": False}, + "transcript_too_large": {"http_status": 413, "retryable": False}, + "invalid_request": {"http_status": 422, "retryable": False}, + "invalid_catalog": {"http_status": 422, "retryable": False}, + "incomplete_plan": {"http_status": 422, "retryable": False}, + "unsupported_protocol_version": {"http_status": 426, "retryable": False}, + "internal_error": {"http_status": 500, "retryable": True}, +} + + +class ProtocolError(ValueError): + """A local or remote evaluator protocol contract violation.""" + + +class UnsupportedProtocolVersion(ProtocolError): + def __init__(self, received: str) -> None: + super().__init__( + f"unsupported evaluator protocol version {received!r}; " + f"supported major version is {PROTOCOL_VERSION}" + ) + self.received = received + + +def validate_protocol_version(version: str) -> None: + if version != PROTOCOL_VERSION: + raise UnsupportedProtocolVersion(version) + + +class EvaluatorKind(str, Enum): + MANAGED = "managed" + CUSTOMER = "customer" + + +class ResultKind(str, Enum): + SCORE = "score" + METRIC = "metric" + ASSERTION = "assertion" + + +class ExecutionMode(str, Enum): + LOCAL = "local" + PYTHON = "python" + + +class TerminalRunStatus(str, Enum): + SUCCEEDED = "succeeded" + FAILED = "failed" + TIMED_OUT = "timed_out" + CANCELLED = "cancelled" + + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +def _wire(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if hasattr(value, "__dataclass_fields__"): + return {key: _wire(item) for key, item in asdict(value).items()} + if isinstance(value, (list, tuple)): + return [_wire(item) for item in value] + if isinstance(value, dict): + return {key: _wire(item) for key, item in value.items()} + return value + + +class WireModel: + def to_wire(self) -> dict[str, Any]: + return _wire(self) + + +def _string(data: Mapping[str, Any], key: str) -> str: + value = data.get(key) + if not isinstance(value, str): + raise ProtocolError(f"{key} must be a string") + return value + + +def _integer(data: Mapping[str, Any], key: str) -> int: + value = data.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{key} must be an integer") + return value + + +def _list(data: Mapping[str, Any], key: str) -> list[Any]: + value = data.get(key) + if not isinstance(value, list): + raise ProtocolError(f"{key} must be an array") + return value + + +def _object(value: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ProtocolError(f"{field_name} must be an object") + return value + + +def _object_list(data: Mapping[str, Any], key: str) -> tuple[Mapping[str, Any], ...]: + return tuple( + _object(value, f"{key}[{index}]") + for index, value in enumerate(_list(data, key)) + ) + + +def _string_list(data: Mapping[str, Any], key: str) -> tuple[str, ...]: + values = _list(data, key) + for index, value in enumerate(values): + if not isinstance(value, str): + raise ProtocolError(f"{key}[{index}] must be a string") + return tuple(values) + + +def _enum(enum_type: type[_EnumT], data: Mapping[str, Any], key: str) -> _EnumT: + value = _string(data, key) + try: + return enum_type(value) + except ValueError as error: + allowed = ", ".join(repr(item.value) for item in enum_type) + raise ProtocolError(f"{key} must be one of {allowed}") from error + + +def _positive_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value <= 0: + raise ProtocolError(f"{key} must be greater than zero") + return value + + +def _nonnegative_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value < 0: + raise ProtocolError(f"{key} must not be negative") + return value + + +def _optional_string(data: Mapping[str, Any], key: str) -> str | None: + value = data.get(key) + if value is not None and not isinstance(value, str): + raise ProtocolError(f"{key} must be a string or null") + return value + + +@dataclass(frozen=True) +class CatalogDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> CatalogDefinition: + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class RegisterRequest(WireModel): + worker_id: str + sdk_version: str + catalog_revision: str + max_concurrency: int + definitions: tuple[CatalogDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + sdk_version=_string(data, "sdk_version"), + catalog_revision=_string(data, "catalog_revision"), + max_concurrency=_integer(data, "max_concurrency"), + definitions=tuple( + CatalogDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class RegisterResponse(WireModel): + evaluator_instance_id: str + evaluator_kind: EvaluatorKind + heartbeat_interval_seconds: int + lease_duration_seconds: int + poll_interval_seconds: int + claim_limit: int + disabled_definitions: tuple[str, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + evaluator_instance_id=_string(data, "evaluator_instance_id"), + evaluator_kind=_enum(EvaluatorKind, data, "evaluator_kind"), + heartbeat_interval_seconds=_integer(data, "heartbeat_interval_seconds"), + lease_duration_seconds=_integer(data, "lease_duration_seconds"), + poll_interval_seconds=_integer(data, "poll_interval_seconds"), + claim_limit=_integer(data, "claim_limit"), + disabled_definitions=_string_list(data, "disabled_definitions"), + ) + + +@dataclass(frozen=True) +class ClaimRequest(WireModel): + worker_id: str + catalog_revision: str + capacity: int + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + catalog_revision=_string(data, "catalog_revision"), + capacity=_integer(data, "capacity"), + ) + + +@dataclass(frozen=True) +class Assignment(WireModel): + assignment_id: str + lease_generation: int + lease_expires_at: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + trigger_reason: str + event_count: int + transcript_url: str + definitions_url: str = "" + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> Assignment: + return cls( + assignment_id=_string(data, "assignment_id"), + lease_generation=_positive_integer(data, "lease_generation"), + lease_expires_at=_string(data, "lease_expires_at"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + trigger_reason=_string(data, "trigger_reason"), + event_count=_nonnegative_integer(data, "event_count"), + transcript_url=_string(data, "transcript_url"), + definitions_url=str(data.get("definitions_url") or ""), + ) + + +@dataclass(frozen=True) +class AssignmentDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + execution_mode: ExecutionMode = ExecutionMode.LOCAL + condition_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> AssignmentDefinition: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), + condition_source=_optional_string(data, "condition_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class DefinitionsResponse(WireModel): + assignment_id: str + catalog_revision: str + definitions: tuple[AssignmentDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> DefinitionsResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + catalog_revision=_string(data, "catalog_revision"), + definitions=tuple( + AssignmentDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class ClaimResponse(WireModel): + assignments: tuple[Assignment, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + tuple( + Assignment.from_wire(item) for item in _object_list(data, "assignments") + ) + ) + + +@dataclass(frozen=True) +class TranscriptEvent(WireModel): + id: str + ts: str + event_type: str + payload: Mapping[str, Any] + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> TranscriptEvent: + payload = data.get("payload") + if not isinstance(payload, Mapping): + raise ProtocolError("payload must be an object") + return cls( + id=_string(data, "id"), + ts=_string(data, "ts"), + event_type=_string(data, "event_type"), + payload=dict(payload), + ) + + +@dataclass(frozen=True) +class SessionTranscript(WireModel): + assignment_id: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + started_at: str + ended_at: str + event_count: int + events: tuple[TranscriptEvent, ...] + schema_version: str = TRANSCRIPT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SessionTranscript: + version = _string(data, "schema_version") + if version != TRANSCRIPT_SCHEMA_VERSION: + raise ProtocolError(f"unsupported transcript schema version {version!r}") + events = tuple( + TranscriptEvent.from_wire(item) for item in _object_list(data, "events") + ) + event_count = _nonnegative_integer(data, "event_count") + if event_count != len(events): + raise ProtocolError( + f"event_count is {event_count}, but transcript contains {len(events)} events" + ) + return cls( + assignment_id=_string(data, "assignment_id"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + started_at=_string(data, "started_at"), + ended_at=_string(data, "ended_at"), + event_count=event_count, + events=events, + ) + + def events_of_type(self, event_type: str) -> tuple[TranscriptEvent, ...]: + return tuple(event for event in self.events if event.event_type == event_type) + + def count(self, event_type: str) -> int: + return sum(event.event_type == event_type for event in self.events) + + +@dataclass(frozen=True) +class EvalSelection(WireModel): + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> EvalSelection: + return cls(_string(data, "eval_key"), _string(data, "eval_version")) + + +@dataclass(frozen=True) +class SkippedEval(WireModel): + eval_key: str + eval_version: str + reason_code: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SkippedEval: + return cls( + _string(data, "eval_key"), + _string(data, "eval_version"), + _string(data, "reason_code"), + ) + + +@dataclass(frozen=True) +class PlanRequest(WireModel): + worker_id: str + lease_generation: int + selected: tuple[EvalSelection, ...] = () + skipped: tuple[SkippedEval, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + selected=tuple( + EvalSelection.from_wire(item) for item in _object_list(data, "selected") + ), + skipped=tuple( + SkippedEval.from_wire(item) for item in _object_list(data, "skipped") + ), + ) + + +@dataclass(frozen=True) +class PlannedRun(WireModel): + evaluation_run_id: str + eval_key: str + eval_version: str + execution_mode: ExecutionMode = ExecutionMode.LOCAL + evaluator_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + eval_key=_string(data, "eval_key"), + eval_version=_string(data, "eval_version"), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), + evaluator_source=_optional_string(data, "evaluator_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class PlanResponse(WireModel): + assignment_id: str + assignment_status: str + runs: tuple[PlannedRun, ...] + idempotent_replay: bool = False + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay", False) + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + assignment_id=_string(data, "assignment_id"), + assignment_status=_string(data, "assignment_status"), + runs=tuple( + PlannedRun.from_wire(item) for item in _object_list(data, "runs") + ), + idempotent_replay=replay, + ) + + +@dataclass(frozen=True) +class HeartbeatRun(WireModel): + evaluation_run_id: str + state: str + progress: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRun: + progress = data.get("progress") + if progress is not None: + if isinstance(progress, bool) or not isinstance(progress, (int, float)): + raise ProtocolError("progress must be a number or null") + progress = float(progress) + if not math.isfinite(progress) or not 0 <= progress <= 1: + raise ProtocolError("progress must be finite and between 0 and 1") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + state=_string(data, "state"), + progress=progress, + ) + + +@dataclass(frozen=True) +class HeartbeatRequest(WireModel): + worker_id: str + lease_generation: int + runs: tuple[HeartbeatRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + runs=tuple( + HeartbeatRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatResponse(WireModel): + lease_expires_at: str + accepted_run_ids: tuple[str, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + lease_expires_at=_string(data, "lease_expires_at"), + accepted_run_ids=_string_list(data, "accepted_run_ids"), + ) + + +@dataclass(frozen=True) +class ResultItem(WireModel): + result_key: str + result_kind: ResultKind + numeric_value: float | None = None + bool_value: bool | None = None + text_value: str | None = None + unit: str = "" + display_value: str | None = None + description: str | None = None + reasoning: str | None = None + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultItem: + numeric = data.get("numeric_value") + if numeric is not None: + if isinstance(numeric, bool) or not isinstance(numeric, (int, float)): + raise ProtocolError("numeric_value must be a number or null") + numeric = float(numeric) + if not math.isfinite(numeric): + raise ProtocolError("numeric_value must be finite") + boolean = data.get("bool_value") + if boolean is not None and not isinstance(boolean, bool): + raise ProtocolError("bool_value must be a boolean or null") + return cls( + result_key=_string(data, "result_key"), + result_kind=_enum(ResultKind, data, "result_kind"), + numeric_value=numeric, + bool_value=boolean, + text_value=_optional_string(data, "text_value"), + unit=_string(data, "unit"), + display_value=_optional_string(data, "display_value"), + description=_optional_string(data, "description"), + reasoning=_optional_string(data, "reasoning"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class ResultRequest(WireModel): + submission_id: str + worker_id: str + lease_generation: int + status: TerminalRunStatus + started_at: str + finished_at: str + duration_ms: int + summary: str | None + results: tuple[ResultItem, ...] + error_code: str | None + error_message: str | None + protocol_version: str = PROTOCOL_VERSION + result_schema_version: str = RESULT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultRequest: + validate_protocol_version(_string(data, "protocol_version")) + result_schema_version = _string(data, "result_schema_version") + if result_schema_version != RESULT_SCHEMA_VERSION: + raise ProtocolError( + f"unsupported result schema version {result_schema_version!r}" + ) + return cls( + submission_id=_string(data, "submission_id"), + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + status=_enum(TerminalRunStatus, data, "status"), + started_at=_string(data, "started_at"), + finished_at=_string(data, "finished_at"), + duration_ms=_nonnegative_integer(data, "duration_ms"), + summary=_optional_string(data, "summary"), + results=tuple( + ResultItem.from_wire(item) for item in _object_list(data, "results") + ), + error_code=_optional_string(data, "error_code"), + error_message=_optional_string(data, "error_message"), + ) + + +@dataclass(frozen=True) +class ResultResponse(WireModel): + evaluation_run_id: str + submission_id: str + status: str + idempotent_replay: bool + result_count: int + result_checksum: str + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay") + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + submission_id=_string(data, "submission_id"), + status=_string(data, "status"), + idempotent_replay=replay, + result_count=_nonnegative_integer(data, "result_count"), + result_checksum=_string(data, "result_checksum"), + ) + + +@dataclass(frozen=True) +class RemoteError(WireModel): + code: str + message: str + retryable: bool + request_id: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RemoteError: + retryable = data.get("retryable") + if not isinstance(retryable, bool): + raise ProtocolError("retryable must be a boolean") + return cls( + code=_string(data, "code"), + message=_string(data, "message"), + retryable=retryable, + request_id=_string(data, "request_id"), + ) + + +@dataclass(frozen=True) +class ErrorResponse(WireModel): + error: RemoteError + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ErrorResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls(RemoteError.from_wire(_object(data.get("error"), "error"))) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py new file mode 100644 index 00000000..0dba852c --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -0,0 +1,863 @@ +"""Async worker state machine for Evaluator v2.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import inspect +import logging +import os +import socket +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from failproofai_sdk import __version__ +from failproofai_sdk.evaluator.authoring import ( + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + DEFAULT_POLL_INTERVAL_SECONDS, + MAX_CLAIM_CAPACITY, + MAX_ERROR_MESSAGE_BYTES, + MAX_WORKER_ID_BYTES, + Assignment, + AssignmentDefinition, + ClaimRequest, + EvalSelection, + ExecutionMode, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + RegisterRequest, + ResultRequest, + SkippedEval, + TerminalRunStatus, +) +from failproofai_sdk.evaluator.source import ( + EvaluationTimeout, + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + +logger = logging.getLogger("failproofai_sdk.evaluator") + +# A synchronous evaluator (or condition) that overruns its timeout cannot be +# cancelled: the executor thread runs the customer function to completion no +# matter what `asyncio.wait_for` does, because CPython cannot interrupt a running +# thread. To stop one such orphaned thread from starving live capacity, the eval +# executor is sized with headroom OVER the concurrency limit — the semaphore, not +# the thread pool, stays the real bound on how many evaluations run at once. This +# is a finite cushion, not a cure: a permanently-blocked synchronous evaluator +# invoked once per session leaks one thread per session, and no fixed pool +# survives that. `sync_evaluations_orphaned` and a warning make the offending +# evaluator findable; prefer `async def` evaluators (cooperatively cancellable) or +# managed PYTHON evaluators (subprocess-isolated, hard-killed) for long or +# untrusted work. +_EVAL_EXECUTOR_ORPHAN_HEADROOM = 8 + +# Reserved out of the lease for the plan request (and network jitter) so the +# pre-plan condition phase always leaves time to submit the plan before the lease +# expires. See `WorkerRuntime._condition_phase_deadline`. +_CONDITION_PHASE_SAFETY_MARGIN_SECONDS = 5.0 + + +def _utc_now() -> str: + return ( + datetime.now(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def _deferred_managed_eval(source: str, timeout_seconds: int | None, eval_key: str): + """Compile server-authored source lazily, at invocation time. + + Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``). + Building the definition with this thunk instead of a pre-compiled function + routes that failure through the same per-run ``try/except`` that turns any + evaluation error into a bounded ``FAILED`` result — so a poison definition + dead-letters cleanly as one failed run instead of raising out of assignment + setup, crashing the task, and forcing the whole assignment to be reclaimed + and retried until its attempt budget is exhausted. + """ + + def evaluate(session: Any) -> Any: + return compile_evaluator( + source, timeout_seconds=timeout_seconds, eval_key=eval_key + )(session) + + return evaluate + + +def _positive_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + if value <= 0: + raise ValueError(f"{name} must be greater than zero") + return value + + +def _boolean(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + normalized = raw.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + +@dataclass(frozen=True) +class WorkerConfig: + server_url: str + credential: str + worker_id: str + max_concurrency: int = 1 + request_timeout_seconds: int = 30 + drain_timeout_seconds: int = 60 + allow_insecure_http: bool = False + + @classmethod + def from_env(cls) -> WorkerConfig: + server_url = os.environ.get("FAILPROOFAI_EVALUATOR_URL", "").strip() + credential = os.environ.get("FAILPROOFAI_EVALUATOR_TOKEN", "").strip() + if not server_url: + raise ValueError("FAILPROOFAI_EVALUATOR_URL is required") + if not credential: + raise ValueError("FAILPROOFAI_EVALUATOR_TOKEN is required") + worker_id = os.environ.get("FAILPROOFAI_EVALUATOR_WORKER_ID", "").strip() + if not worker_id: + worker_id = f"{socket.gethostname()}-{os.getpid()}" + if len(worker_id.encode("utf-8")) > MAX_WORKER_ID_BYTES: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_WORKER_ID exceeds {MAX_WORKER_ID_BYTES} bytes" + ) + if any(ord(character) < 32 or ord(character) == 127 for character in worker_id): + raise ValueError( + "FAILPROOFAI_EVALUATOR_WORKER_ID must not contain control characters" + ) + config = cls( + server_url=server_url, + credential=credential, + worker_id=worker_id, + max_concurrency=_positive_int("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), + request_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30 + ), + drain_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 + ), + allow_insecure_http=_boolean( + "FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP" + ), + ) + if config.max_concurrency > MAX_CLAIM_CAPACITY: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds {MAX_CLAIM_CAPACITY}" + ) + return config + + +class WorkerRuntime: + def __init__( + self, + evaluator: Evaluator, + config: WorkerConfig, + *, + client: EvaluatorClient | None = None, + ) -> None: + self.evaluator = evaluator + self.config = config + self.client = client or EvaluatorClient( + base_url=config.server_url, + credential=config.credential, + timeout_seconds=config.request_timeout_seconds, + allow_insecure_http=config.allow_insecure_http, + ) + self._stopping = asyncio.Event() + self._active: set[asyncio.Task[None]] = set() + self._heartbeat_interval = 30 + self._poll_interval = DEFAULT_POLL_INTERVAL_SECONDS + self._claim_limit = config.max_concurrency + self._lease_duration = 120 + self._disabled_definitions: set[str] = set() + self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + # Headroom over the semaphore so a timed-out-but-still-running synchronous + # evaluator (an unkillable orphaned thread) does not immediately starve + # live capacity — the semaphore remains the true concurrency bound. See + # `_EVAL_EXECUTOR_ORPHAN_HEADROOM`. + self._eval_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=config.max_concurrency + _EVAL_EXECUTOR_ORPHAN_HEADROOM, + thread_name_prefix="failproof-eval", + ) + self._registered = False + self._last_server_contact: float | None = None + self._metric_lock = threading.Lock() + self._metrics: dict[str, int] = {} + + async def register(self) -> None: + try: + response = await self._call_client( + self.client.register, + RegisterRequest( + worker_id=self.config.worker_id, + sdk_version=__version__, + catalog_revision=self.evaluator.catalog_revision, + max_concurrency=self.config.max_concurrency, + definitions=self.evaluator.catalog(), + ), + ) + except Exception: + self._increment("registration_failure") + raise + self._heartbeat_interval = response.heartbeat_interval_seconds + self._poll_interval = response.poll_interval_seconds + self._lease_duration = response.lease_duration_seconds + self._claim_limit = min(self.config.max_concurrency, response.claim_limit) + if ( + self._heartbeat_interval <= 0 + or self._poll_interval <= 0 + or self._lease_duration <= self._heartbeat_interval + or self._claim_limit <= 0 + ): + self._increment("registration_failure") + raise RuntimeError( + "server returned invalid evaluator timing or claim limits" + ) + self._disabled_definitions = set(response.disabled_definitions) + self._registered = True + self._increment("registration_success") + + async def run_forever(self) -> None: + await self.register() + retry_delay = 1.0 + try: + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + delay = ( + float(self._lease_duration) + if error.status is None + else retry_delay + ) + await self._wait_or_stop(delay) + retry_delay = min(retry_delay * 2.0, 30.0) + continue + retry_delay = 1.0 + assignments = self._validated_assignments( + response.assignments, capacity + ) + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + if not assignments: + # Normal short poll: the server returns immediately, so when + # nothing is queued we wait the advertised interval before + # polling again instead of hot-looping. When work IS returned + # we loop straight back to drain any backlog up to capacity. + await self._wait_or_stop(float(self._poll_interval)) + finally: + await self.drain() + self._eval_executor.shutdown(wait=False, cancel_futures=True) + + async def run_once(self) -> int: + """Claim once and finish the returned assignments; useful for jobs/tests.""" + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=self._claim_limit, + ), + ) + assignments = self._validated_assignments( + response.assignments, self._claim_limit + ) + self._increment("assignments_claimed", len(assignments)) + await asyncio.gather(*(self.process_assignment(item) for item in assignments)) + return len(assignments) + + def stop(self) -> None: + self._stopping.set() + + async def drain(self) -> None: + self._reap_finished() + if not self._active: + return + done, pending = await asyncio.wait( + self._active, timeout=self.config.drain_timeout_seconds + ) + for task in done: + self._consume_task(task) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + self._active.clear() + + async def process_assignment(self, assignment: Assignment) -> None: + session = await self._call_client( + self.client.transcript, + assignment, + worker_id=self.config.worker_id, + ) + if session.session_revision_id != assignment.session_revision_id: + raise RuntimeError("transcript session revision does not match assignment") + + descriptors = await self._assignment_definitions(assignment) + selected: list[tuple[AssignmentDefinition, EvalDefinition | None]] = [] + skipped: list[SkippedEval] = [] + local_definitions = { + (item.eval_key, item.eval_version): item + for item in self.evaluator.definitions + } + # The assignment lease is fixed at claim time and cannot be renewed until + # the plan is submitted (the server only extends a lease for a *planned* + # assignment with running runs). A slow condition phase can therefore burn + # the whole lease and get the plan fenced as lease_lost, so every + # condition is bounded by the lease it must leave time to plan within. + condition_deadline = self._condition_phase_deadline(assignment) + for descriptor in descriptors: + local = local_definitions.get( + (descriptor.eval_key, descriptor.eval_version) + ) + if descriptor.execution_mode is ExecutionMode.LOCAL and local is None: + raise RuntimeError("server requested a definition absent from this worker") + if descriptor.eval_key in self._disabled_definitions: + skipped.append(self._skipped_descriptor(descriptor, "disabled_by_server")) + self._increment("conditions_skipped") + continue + try: + # Whose condition decides applicability follows the execution mode, + # mirroring the evaluator branch below (`run.execution_mode`): a LOCAL + # definition's condition is client-authored (`local.condition`); a + # PYTHON (managed) definition's is server-authored and MUST govern even + # when the worker also registered the same key/version locally. Keying + # `local` on `(eval_key, eval_version)` alone means a managed def can + # collide with a local one; selecting `local.condition` there would let + # a matching local condition override the server's managed rule and run + # the managed evaluator against the operator's intent (COR-001). + # Compile INSIDE the try: a managed condition the sandbox rejects + # (unsafe/malformed source) must dead-letter as `condition_error`, + # not raise out of the plan loop and strand the whole assignment + # until its retry budget is exhausted. + managed_condition_source: str | None = None + if descriptor.execution_mode is ExecutionMode.LOCAL: + condition_function = local.condition if local is not None else None + elif descriptor.condition_source: + # Compiled below, once the lease budget is known, so the + # sandbox subprocess is bounded by whatever lease remains. + managed_condition_source = descriptor.condition_source + condition_function = None + else: + condition_function = None + if condition_function is None and managed_condition_source is None: + # No condition to run — applicable by default, no lease spent. + selected.append((descriptor, local)) + continue + budget = self._condition_budget( + condition_deadline, descriptor.timeout_seconds + ) + if budget <= 0.0: + # Not enough lease left to evaluate this condition and still + # submit the plan in time; skip it (and, as the loop proceeds, + # every later condition) rather than do work the server will + # fence as lease_lost and reclaim in a loop. + skipped.append( + self._skipped_descriptor(descriptor, "lease_exhausted") + ) + self._increment("conditions_skipped") + self._increment("conditions_lease_exhausted") + continue + if managed_condition_source is not None: + condition_function = compile_condition( + managed_condition_source, + timeout_seconds=budget, + ) + condition = await asyncio.wait_for( + self._invoke(condition_function, session), timeout=budget + ) + if isinstance(condition, ConditionResult): + applicable = condition.applicable + reason_code = condition.reason_code + elif isinstance(condition, bool): + applicable = condition + reason_code = "condition_false" + else: + raise TypeError("condition must return bool or ConditionResult") + except Exception as error: # noqa: BLE001 - isolates customer condition code + logger.warning( + "evaluator condition failed", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + skipped.append(self._skipped_descriptor(descriptor, "condition_error")) + self._increment("conditions_skipped") + continue + if applicable: + selected.append((descriptor, local)) + self._increment("conditions_selected") + else: + skipped.append(self._skipped_descriptor(descriptor, reason_code)) + self._increment("conditions_skipped") + + plan = await self._call_client( + self.client.plan, + assignment.assignment_id, + PlanRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + selected=tuple( + EvalSelection(item.eval_key, item.eval_version) + for item, _local in selected + ), + skipped=tuple(skipped), + ), + ) + if plan.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned a plan for a different assignment") + expected_status = "planned" if selected else "skipped" + if plan.assignment_status != expected_status: + raise RuntimeError("server returned an inconsistent assignment status") + + definitions = { + (item.eval_key, item.eval_version): (item, local) + for item, local in selected + } + run_definitions: list[tuple[str, EvalDefinition]] = [] + run_ids: set[str] = set() + for run in plan.runs: + if run.evaluation_run_id in run_ids: + raise RuntimeError("server returned a duplicate evaluation run id") + run_ids.add(run.evaluation_run_id) + selected_definition = definitions.pop( + (run.eval_key, run.eval_version), None + ) + if selected_definition is None: + raise RuntimeError("server returned an unrequested evaluation run") + descriptor, local = selected_definition + if run.execution_mode is not descriptor.execution_mode: + raise RuntimeError("server changed the evaluation execution mode") + if run.execution_mode is ExecutionMode.LOCAL: + if local is None: + raise RuntimeError("local evaluation definition is unavailable") + definition = local + else: + if not run.evaluator_source or not run.source_checksum: + raise RuntimeError("server omitted managed evaluation source") + expected = source_checksum( + descriptor.condition_source, run.evaluator_source + ) + if expected != run.source_checksum or ( + descriptor.source_checksum + and descriptor.source_checksum != run.source_checksum + ): + raise RuntimeError("managed evaluation source checksum mismatch") + definition = EvalDefinition( + eval_key=descriptor.eval_key, + display_name=descriptor.display_name, + eval_version=descriptor.eval_version, + result_kind=descriptor.result_kind, + labels=descriptor.labels, + function=_deferred_managed_eval( + run.evaluator_source, + run.timeout_seconds or descriptor.timeout_seconds, + descriptor.eval_key, + ), + condition=None, + on_cancel=None, + timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, + ) + run_definitions.append((run.evaluation_run_id, definition)) + if definitions and not plan.idempotent_replay: + raise RuntimeError("server omitted a selected evaluation run") + + tasks = { + run_id: asyncio.create_task( + self._execute_run(assignment, run_id, definition, session) + ) + for run_id, definition in run_definitions + } + heartbeat = asyncio.create_task(self._heartbeat(assignment, tasks)) + try: + outcomes = await asyncio.gather(*tasks.values(), return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise outcome + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _execute_run( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + async with self._eval_semaphore: + await self._execute_run_in_slot(assignment, run_id, definition, session) + + async def _execute_run_in_slot( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + started_at = _utc_now() + started = time.monotonic() + # A synchronous evaluator runs in the executor thread; if it overruns the + # wall-clock timeout below, the thread cannot be cancelled and is orphaned. + sync_function = not inspect.iscoroutinefunction(definition.function) + try: + invocation = self._invoke(definition.function, session) + result = ( + await asyncio.wait_for(invocation, timeout=definition.timeout_seconds) + if definition.timeout_seconds is not None + else await invocation + ) + if not isinstance(result, EvalResult): + raise TypeError("evaluation must return EvalResult") + items = result.result_items(definition.eval_key) + if not any( + item.result_key == definition.eval_key + and item.result_kind == definition.result_kind + for item in items + ): + raise ValueError( + "evaluation result does not contain its declared primary result" + ) + status = TerminalRunStatus.SUCCEEDED + summary = result.summary + error_code = None + error_message = None + except (asyncio.TimeoutError, EvaluationTimeout) as timeout_error: + # asyncio.TimeoutError: the awaiter hit the wall-clock. EvaluationTimeout: + # the forked managed sandbox was killed by its CPU/memory/time budget — + # the real, thread-uncancellable case. Both are a timed-out run. + await self._cancel_hook(definition, session) + if isinstance(timeout_error, asyncio.TimeoutError) and sync_function: + # The awaiter gave up while a SYNCHRONOUS evaluator was still + # running in the executor. CPython cannot interrupt that thread, + # so it is now orphaned — it runs to completion (or forever) + # holding a worker thread. Count it and name the evaluator so a + # hung one is findable; the executor's headroom keeps this one + # orphan from immediately starving live capacity. + self._increment("sync_evaluations_orphaned") + logger.warning( + "synchronous evaluation exceeded its timeout and cannot be " + "cancelled; its worker thread is orphaned until it returns", + extra={ + "assignment_id": assignment.assignment_id, + "eval_key": definition.eval_key, + }, + ) + items = () + status = TerminalRunStatus.TIMED_OUT + summary = None + error_code = "eval_timeout" + error_message = "evaluation exceeded its configured timeout" + except asyncio.CancelledError: + await self._cancel_hook(definition, session) + raise + except UnsafeEvaluatorSource as error: + # Surface the REASON for a rejected server-authored definition. + # + # This is deliberately narrower than the generic handler below. + # UnsafeEvaluatorSource is raised by our own validator before any + # customer source executes, and its message is SDK-authored text + # about the source's shape ("evaluator_source must be one + # expression", "contains disallowed syntax: Assign") — it embeds no + # transcript content, so it is safe to send back over the wire. + # + # Without this the author saw only "evaluation raised + # UnsafeEvaluatorSource" on every session, with no way to learn what + # was wrong: the server accepts any source that passes its size and + # key checks, so a definition that can never run is published + # successfully and then fails silently and permanently. + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + detail = str(error).strip() + error_message = ( + f"evaluator source rejected: {detail}" + if detail + else "evaluator source rejected by the sandbox validator" + ) + encoded = error_message.encode("utf-8") + if len(encoded) > MAX_ERROR_MESSAGE_BYTES: + error_message = encoded[:MAX_ERROR_MESSAGE_BYTES].decode( + "utf-8", "ignore" + ) + except Exception as error: # noqa: BLE001 - converts customer eval failures + # Type name ONLY. A customer eval's exception text can quote the + # transcript it was reading, and this field is persisted and shown + # in the dashboard, so the message itself is not repeated here. + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + error_message = f"evaluation raised {type(error).__name__}" + + request = ResultRequest( + submission_id=str(uuid4()), + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + status=status, + started_at=started_at, + finished_at=_utc_now(), + duration_ms=max(0, round((time.monotonic() - started) * 1_000)), + summary=summary, + results=items, + error_code=error_code, + error_message=error_message, + ) + await self._call_client(self.client.submit_result, run_id, request) + self._increment(f"runs_{status.value}") + + async def _cancel_hook(self, definition: EvalDefinition, session: Any) -> None: + if definition.on_cancel is None: + return + try: + await self._invoke(definition.on_cancel, session) + except Exception as error: # noqa: BLE001 - cancellation hooks are customer code + logger.warning( + "evaluator cancellation hook failed", + extra={"error_type": type(error).__name__}, + ) + + async def _heartbeat( + self, + assignment: Assignment, + tasks: dict[str, asyncio.Task[None]], + ) -> None: + while True: + await asyncio.sleep(self._heartbeat_interval) + active = tuple( + HeartbeatRun(evaluation_run_id=run_id, state="running") + for run_id, task in tasks.items() + if not task.done() + ) + if not active: + return + try: + response = await self._call_client( + self.client.heartbeat, + HeartbeatRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + runs=active, + ), + ) + accepted = set(response.accepted_run_ids) + for run_id, task in tasks.items(): + if not task.done() and run_id not in accepted: + task.cancel() + except EvaluatorAPIError as error: + if error.code == "lease_lost": + self._increment("leases_lost") + for task in tasks.values(): + task.cancel() + return + logger.warning( + "evaluator heartbeat failed", + extra={ + "assignment_id": assignment.assignment_id, + "code": error.code, + }, + ) + self._increment("heartbeat_failures") + except Exception as error: # noqa: BLE001 - keep lease renewal alive + logger.warning( + "evaluator heartbeat error", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + self._increment("heartbeat_failures") + + def _condition_phase_deadline(self, assignment: Assignment) -> float: + """Monotonic-clock reading by which the pre-plan condition phase must end. + + The real `lease_expires_at` is used when it is in the future (production); + a past or unparseable value (clock skew, or a replayed transcript in a + test) falls back to the negotiated lease duration measured from now, so + the bound never fires spuriously on a stale deadline. + """ + remaining = float(self._lease_duration) + try: + expires = datetime.fromisoformat( + assignment.lease_expires_at.replace("Z", "+00:00") + ) + parsed = (expires - datetime.now(timezone.utc)).total_seconds() + if parsed > 0: + remaining = parsed + except (ValueError, AttributeError): + pass + return time.monotonic() + remaining + + def _condition_budget( + self, deadline: float, timeout_seconds: int | None + ) -> float: + """Seconds a single condition may run: the lease left before the + plan-submission margin, capped by the definition's own timeout.""" + remaining = ( + deadline - time.monotonic() - _CONDITION_PHASE_SAFETY_MARGIN_SECONDS + ) + if timeout_seconds is not None: + remaining = min(remaining, float(timeout_seconds)) + return remaining + + async def _invoke(self, function, session): + if inspect.iscoroutinefunction(function): + return await function(session) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(self._eval_executor, function, session) + if inspect.isawaitable(result): + return await result + return result + + @staticmethod + def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + @staticmethod + def _skipped_descriptor( + definition: AssignmentDefinition, reason: str + ) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + async def _assignment_definitions( + self, assignment: Assignment + ) -> tuple[AssignmentDefinition, ...]: + if assignment.definitions_url: + response = await self._call_client( + self.client.definitions, + assignment, + worker_id=self.config.worker_id, + ) + if response.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned definitions for another assignment") + return response.definitions + return tuple( + AssignmentDefinition( + eval_key=item.eval_key, + display_name=item.display_name, + eval_version=item.eval_version, + result_kind=item.result_kind, + labels=item.labels, + ) + for item in self.evaluator.definitions + ) + + def _reap_finished(self) -> None: + done = {task for task in self._active if task.done()} + self._active.difference_update(done) + for task in done: + self._consume_task(task) + + @staticmethod + def _validated_assignments( + assignments: tuple[Assignment, ...], capacity: int + ) -> tuple[Assignment, ...]: + if len(assignments) > capacity: + raise RuntimeError("server returned more assignments than requested") + assignment_ids = [item.assignment_id for item in assignments] + if len(assignment_ids) != len(set(assignment_ids)): + raise RuntimeError("server returned duplicate assignments") + return assignments + + @staticmethod + def _consume_task(task: asyncio.Task[None]) -> None: + try: + task.result() + except asyncio.CancelledError: + pass + except Exception: + logger.exception("evaluator assignment failed") + + async def _wait_for_progress(self) -> None: + if not self._active: + return + stop_task = asyncio.create_task(self._stopping.wait()) + try: + await asyncio.wait( + (*self._active, stop_task), return_when=asyncio.FIRST_COMPLETED + ) + finally: + if not stop_task.done(): + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + + async def _wait_or_stop(self, seconds: float) -> None: + try: + await asyncio.wait_for(self._stopping.wait(), timeout=seconds) + except asyncio.TimeoutError: + pass + + async def _call_client(self, function, *args, **kwargs): + result = await asyncio.to_thread(function, *args, **kwargs) + self._last_server_contact = time.monotonic() + return result + + def _increment(self, name: str, amount: int = 1) -> None: + with self._metric_lock: + self._metrics[name] = self._metrics.get(name, 0) + amount + + def metrics(self) -> dict[str, int]: + with self._metric_lock: + return dict(self._metrics) + + def is_ready(self) -> bool: + if ( + self._stopping.is_set() + or not self._registered + or self._last_server_contact is None + ): + return False + return time.monotonic() - self._last_server_contact <= max( + float(self._lease_duration), 60.0 + ) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py new file mode 100644 index 00000000..8327c7f8 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -0,0 +1,681 @@ +"""Restricted deterministic expression compiler for server-authored evaluations. + +The managed worker never executes a module, statements, imports, or ambient +builtins from tenant-authored source. Definitions are single Python expressions +evaluated with a small constructor/helper surface and the immutable transcript +bound as ``session``. +""" + +from __future__ import annotations + +import ast +import builtins +import hashlib +import os +import pickle +import re +import select +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Callable +from typing import Any + +try: + import resource as _resource +except ImportError: # pragma: no cover - non-POSIX + _resource = None # type: ignore[assignment] + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalResult, + Metric, + Score, +) + +MAX_CONDITION_SOURCE_BYTES = 16 * 1024 +MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024 + +# Static defense-in-depth bounds applied at COMPILE time (see `_compile`). They +# reject the obvious authoring bombs early; they are NOT the primary defense — +# a runtime-computed size (`range(len(session.events) ** 40)`) slips past any +# static check, which is exactly why the fork sandbox below is the real bound. +MAX_AST_NODES = 5_000 +MAX_POW_EXPONENT = 64 + +# Hard ceilings for ONE sandboxed evaluation, enforced by the kernel in a +# fork+exec'd subprocess (see `_run_sandboxed`). RLIMIT_CPU + the parent's +# wall-clock kill both bound compute bombs; RLIMIT_AS is the memory backstop for a +# giant-int / huge-allocation bomb. +DEFAULT_SANDBOX_TIMEOUT_SECONDS = 30 +# The effective budget is CLAMPED to this ceiling regardless of the (server-set) +# per-definition timeout, so a large `timeout_seconds` can never remove the +# execution bound (SEC-001). Wall-clock and CPU are both capped here. +MAX_SANDBOX_TIMEOUT_SECONDS = 60 +# Per-sandbox address-space cap. A managed eval works over a transcript (<=25 MiB) +# and returns a small result, so this is generous; it also rejects an allocation +# bomb (`[0] * 200000000` is ~1.6 GiB > this) before it returns a valid result. +SANDBOX_MEMORY_BYTES = 512 * 1024 * 1024 # 512 MiB +# ...but a per-process cap alone does not bound the HOST: a worker with +# max_concurrency=32 could run 32 sandboxes at once. Cap the number of concurrent +# sandbox processes so the AGGREGATE (MAX_CONCURRENT_SANDBOXES * SANDBOX_MEMORY_BYTES, +# ~2 GiB) is bounded independent of the worker's claim concurrency; extra evals +# queue on the semaphore rather than pile up memory. +MAX_CONCURRENT_SANDBOXES = 4 +_SANDBOX_SLOTS = threading.Semaphore(MAX_CONCURRENT_SANDBOXES) +# The result crossing back is bounded on BOTH sides: the child refuses to serialize +# a result larger than this, and the parent stops reading (and kills the child) +# past it — so a permitted expression that builds a huge result +# (`EvalResult(metrics={str(x): 1 for x in range(100000)})`) cannot OOM the worker +# even though the child's RLIMIT_AS lets it construct one. A valid result (<=25 +# items, bounded fields) is far under this. +SANDBOX_MAX_RESULT_BYTES = 1 * 1024 * 1024 # 1 MiB + +# The sandbox child is scrubbed of the worker's environment. +# +# `subprocess.Popen` inherits `os.environ` by default, which on a worker means +# FAILPROOFAI_EVALUATOR_TOKEN — and on the FailproofAI-managed pod that token is +# the CROSS-TENANT credential the whole fleet authenticates with. The AST and +# empty-builtins restrictions already stop a managed expression from reading +# `os.environ`, so this is defence in depth rather than a fix for a live escape: +# it means a future gap in those restrictions cannot be escalated into credential +# theft. Only the variables the interpreter itself needs are forwarded — notably +# PYTHONPATH, without which the child cannot import the sandbox runner at all. +_SANDBOX_ENV_PASSTHROUGH = ( + "PATH", + "PYTHONPATH", + "PYTHONHOME", + "PYTHONDONTWRITEBYTECODE", + "PYTHONUNBUFFERED", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "SYSTEMROOT", # Windows: CPython fails to start without it +) + + +def _sandbox_env() -> dict[str, str]: + return { + name: os.environ[name] + for name in _SANDBOX_ENV_PASSTHROUGH + if os.environ.get(name) + } + + +def _clamp_budget(timeout_seconds: float | None) -> float: + """The wall-clock/CPU budget for one evaluation: a positive value no larger + than MAX_SANDBOX_TIMEOUT_SECONDS. Server-provided timeouts cannot exceed it.""" + requested = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) + if requested <= 0: + requested = DEFAULT_SANDBOX_TIMEOUT_SECONDS + return min(requested, float(MAX_SANDBOX_TIMEOUT_SECONDS)) + + +class EvaluationTimeout(Exception): + """A sandboxed evaluation exceeded its CPU/memory/wall-clock budget. + + Distinct from an eval that *returned* an error: the computation was forcibly + terminated because it could not be allowed to keep running. + """ + + +class EvaluationSandboxUnavailable(Exception): + """The killable-process sandbox could not be established. + + Raised instead of running server-authored source unsandboxed — if the sandbox + subprocess cannot be started, or the transcript cannot be serialized into it, + there is no way to bound or terminate the evaluation, so we fail closed + (SEC-001). + """ + + +def _install_limits(cpu_seconds: float, mem_bytes: int) -> None: + """Install hard CPU + address-space limits on the CURRENT process. + + Called by the sandbox subprocess on itself, right before it evaluates. + """ + if _resource is None: # pragma: no cover - non-POSIX + return + cpu = max(1, int(cpu_seconds)) + _resource.setrlimit(_resource.RLIMIT_CPU, (cpu, cpu)) + _resource.setrlimit(_resource.RLIMIT_AS, (mem_bytes, mem_bytes)) + + +def _run_sandboxed( + kind: str, + source: str, + session: Any, + *, + wall_timeout: float, + cpu_seconds: float, + mem_bytes: int, + eval_key: str | None = None, +) -> Any: + """Evaluate server-authored ``source`` against ``session`` in a fork+exec'd + subprocess that CANNOT outlive its budget or flood this process. + + A FRESH ``python -m ..._sandbox_runner`` process — never a fork of this + multi-threaded worker (forking one deadlocks the child on a lock some other + thread holds) — reads its input from a temp file, installs hard RLIMIT_CPU + + RLIMIT_AS on itself, evaluates, and writes a bounded result to stdout. This + parent reads stdout up to ``SANDBOX_MAX_RESULT_BYTES`` and no further, killing + the child on timeout OR oversize — so neither compute (``sum(range(10**20))``) + nor an oversized result (``metrics={str(x):1 for x in range(100000)}``) can + exhaust the worker. + """ + # SEC-001: without the stdlib ``resource`` module (e.g. Windows) the sandbox + # child cannot install RLIMIT_CPU / RLIMIT_AS on itself (``_install_limits`` + # no-ops), so a permitted expression could allocate unbounded memory before + # the parent's wall-clock kill lands. Refuse BEFORE spawning any child rather + # than run server-authored source without the advertised limits. + if _resource is None: # pragma: no cover - non-POSIX + raise EvaluationSandboxUnavailable( + "kernel resource limits (RLIMIT_CPU/RLIMIT_AS) are unavailable on this " + "platform; managed evaluation cannot be bounded, refusing to run" + ) + try: + session_wire = session.to_wire() + except AttributeError as error: + raise EvaluationSandboxUnavailable( + "sandboxed evaluation requires a serializable transcript" + ) from error + payload = pickle.dumps( + (kind, source, session_wire, cpu_seconds, mem_bytes, eval_key) + ) + # Input via a temp file, not stdin: the transcript can be large (up to the + # transcript ceiling) and feeding a big stdin while bounding stdout invites a + # pipe deadlock. The child reads the file; we only read its stdout. + handle, path = tempfile.mkstemp(prefix="fpai-sandbox-", suffix=".pkl") + try: + with os.fdopen(handle, "wb") as tmp: + tmp.write(payload) + chunks: list[bytes] = [] + total = 0 + timed_out = False + too_large = False + # A slot is held for the whole subprocess lifetime so no more than + # MAX_CONCURRENT_SANDBOXES run at once — bounding aggregate memory across + # concurrent sandboxes. But acquiring it must COUNT AGAINST the wall-clock + # budget: the runtime runs this in a thread and `asyncio.wait_for` only + # cancels the awaiter, so a thread that blocked here UNBOUNDED past its + # deadline would still go on to launch a sandbox after its run was already + # reported timed out — 28 such threads could queue behind 4 long sandboxes + # and starve the worker (conditions have no runtime-level wait at all). One + # deadline therefore covers BOTH the slot wait and execution: we acquire the + # slot with the remaining budget and, on failure, time out WITHOUT spawning. + deadline = time.monotonic() + wall_timeout + acquire_timeout = deadline - time.monotonic() + if acquire_timeout <= 0 or not _SANDBOX_SLOTS.acquire(timeout=acquire_timeout): + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") + try: + if deadline - time.monotonic() <= 0: + # Slot acquired exactly at the deadline: a child launched now could + # only be killed immediately, so do not spawn one at all. + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") + try: + proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=_sandbox_env(), + ) + except OSError as error: + raise EvaluationSandboxUnavailable( + f"could not start the evaluation sandbox: {error}" + ) from error + out_fd = proc.stdout.fileno() + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + ready, _, _ = select.select([out_fd], [], [], remaining) + if not ready: + timed_out = True + break + chunk = os.read(out_fd, 65536) + if not chunk: + break + total += len(chunk) + if total > SANDBOX_MAX_RESULT_BYTES: + too_large = True + break + chunks.append(chunk) + finally: + proc.stdout.close() + if proc.poll() is None: + proc.kill() + proc.wait() + finally: + _SANDBOX_SLOTS.release() + finally: + try: + os.unlink(path) + except OSError: + pass + + if timed_out: + raise EvaluationTimeout("evaluation exceeded its wall-clock budget") + if too_large: + raise EvaluationTimeout("evaluation result exceeded the size limit") + data = b"".join(chunks) + if not data: + # Killed by RLIMIT_CPU/RLIMIT_AS (or otherwise died) before it could write. + raise EvaluationTimeout("evaluation was terminated before producing a result") + outcome = pickle.loads(data) + if outcome[0] == "ok": + return outcome[1] + # Preserve the child's original exception SEMANTICS: an eval's + # NameError/TypeError/ZeroDivisionError/... and the sandbox's own + # UnsafeEvaluatorSource must read the same as they did in-process. Reconstruct + # any builtin exception by name; anything else collapses to a generic error — + # still caught as a failed run upstream. + _, name, message = outcome + if name == UnsafeEvaluatorSource.__name__: + raise UnsafeEvaluatorSource(message) + builtin = getattr(builtins, name, None) + if isinstance(builtin, type) and issubclass(builtin, BaseException): + raise builtin(message) + raise RuntimeError(f"{name}: {message}") + +_ALLOWED_NODES = ( + ast.Expression, + ast.BoolOp, + ast.BinOp, + ast.UnaryOp, + ast.IfExp, + ast.Dict, + ast.Set, + ast.List, + ast.Tuple, + ast.ListComp, + ast.SetComp, + ast.DictComp, + # `ast.GeneratorExp` is intentionally NOT allowed: a bare generator object's + # default repr is ``, which leaks a live host + # heap address (an ASLR/memory-layout disclosure) the moment it is coerced to + # a string into any result field. List/set/dict comprehensions render as their + # data (`[...]`, `{...}`) and cover the same ground — wrap a generator in `[]`. + ast.comprehension, + ast.Compare, + ast.Call, + ast.FormattedValue, + ast.JoinedStr, + ast.Constant, + ast.Name, + ast.Load, + ast.Store, + ast.Attribute, + ast.Subscript, + ast.Slice, + ast.keyword, + ast.And, + ast.Or, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.USub, + ast.UAdd, + ast.Not, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, +) + +_SAFE_GLOBALS = { + "__builtins__": {}, + "Assertion": Assertion, + "ConditionResult": ConditionResult, + "EvalResult": EvalResult, + "Metric": Metric, + "Score": Score, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + # `enumerate` is intentionally excluded: an enumerate object's default repr is + # ``, leaking a live host heap address into any + # result field. Index-aware iteration can use `range(len(...))` instead. + "float": float, + "int": int, + "len": len, + "list": list, + "max": max, + "min": min, + "range": range, + "round": round, + "set": set, + "sorted": sorted, + "str": str, + "sum": sum, + "tuple": tuple, +} + + +# Attribute access is DEFAULT-DENY. A denylist is unwinnable here: dunder access +# is only one door. `str.format`/`format_map` traverse a format string's fields +# at the C level; `(x for x in [1]).gi_frame.f_globals` reaches the eval globals +# through generator/frame introspection; `str.mro()[-1]` reaches the `object` +# type — and NONE of `format`, `gi_frame`, `f_globals`, `co_names`, `mro`, ... +# start with an underscore, so the dunder guard never sees them. Rather than +# chase each introspection family, we allow ONLY the attribute names a real +# session evaluation needs: the transcript/event data surface plus a fixed set +# of pure string/collection data methods. Anything else — every current and +# future introspection attribute — is rejected. `format`/`format_map` are simply +# absent from this set, so the C-level format escape is closed too. +# Data attributes on the transcript surface — safe to READ as a value: each is a +# field of a frozen dataclass (SessionTranscript / TranscriptEvent, whose reprs are +# field-based and pointer-free) or a JSON scalar/container from an event payload. +_DATA_ATTRS = frozenset( + { + # SessionTranscript + TranscriptEvent data surface (see protocol.py). + "events", + "event_count", + "event_type", + "payload", + "id", + "ts", + "agent_id", + "environment", + "session_id", + "session_revision_id", + "assignment_id", + "started_at", + "ended_at", + "schema_version", + } +) + +# Method attributes — pure data methods that must be CALLED, never referenced as a +# bare value. A bound method's repr is `<... at 0x...>`, a live heap address; a bare +# reference (`payload.get` uncalled) is only ever useful for smuggling that address +# into a result field via `str()`, an f-string, or `%`-formatting — none of which a +# real evaluation needs. `_compile` requires each of these names to appear at a call +# site, which closes every text-coercion leak at its source: no reachable value can +# then carry a pointer repr, so the output-boundary scan is only defense in depth. +_METHOD_ATTRS = frozenset( + { + # SessionTranscript methods. + "events_of_type", + "count", + # dict data methods. + "get", + "keys", + "values", + "items", + # str / bytes pure data methods. + "lower", + "upper", + "strip", + "lstrip", + "rstrip", + "split", + "rsplit", + "splitlines", + "startswith", + "endswith", + "replace", + "find", + "rfind", + "index", + "join", + "title", + "capitalize", + "casefold", + "swapcase", + "isdigit", + "isalpha", + "isalnum", + "isspace", + "isnumeric", + "isdecimal", + "islower", + "isupper", + "istitle", + "zfill", + "ljust", + "rjust", + "center", + "partition", + "rpartition", + "removeprefix", + "removesuffix", + "encode", + "decode", + "hex", + # set data methods. + "union", + "intersection", + "difference", + "symmetric_difference", + "issubset", + "issuperset", + "isdisjoint", + } +) + +# The walk rejects any attribute outside this union, and additionally requires every +# name in `_METHOD_ATTRS` to appear only as the function of a call. +_ALLOWED_ATTRS = _DATA_ATTRS | _METHOD_ATTRS + + +def _fresh_globals() -> dict[str, Any]: + """A throwaway globals mapping for one eval call. + + Every evaluation gets its own copy — with a fresh empty ``__builtins__`` — + so that even if a future reach exposes the eval's globals (e.g. through a + frame object), a mutation cannot persist into another evaluation and poison + a shared, process-wide namespace. + """ + return {**_SAFE_GLOBALS, "__builtins__": {}} + + +class UnsafeEvaluatorSource(ValueError): + """Raised before any disallowed server-authored source can execute.""" + + +def source_checksum(condition_source: str | None, evaluator_source: str) -> str: + payload = (condition_source or "").encode("utf-8") + b"\0" + evaluator_source.encode( + "utf-8" + ) + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _compile(source: str, *, field_name: str, maximum: int) -> Any: + if not isinstance(source, str) or not source.strip(): + raise UnsafeEvaluatorSource(f"{field_name} must not be empty") + if len(source.encode("utf-8")) > maximum: + raise UnsafeEvaluatorSource(f"{field_name} exceeds {maximum} bytes") + try: + tree = ast.parse(source, mode="eval") + except SyntaxError as error: + raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + # An Attribute that is the function of a Call is a method invocation; any other + # Attribute naming a method (`_METHOD_ATTRS`) is a bare bound-method reference, + # whose only use is leaking the method's `<... at 0xADDR>` repr into a result. + called_method_nodes = { + node.func + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + node_count = 0 + for node in ast.walk(tree): + node_count += 1 + if node_count > MAX_AST_NODES: + raise UnsafeEvaluatorSource( + f"{field_name} is too large ({MAX_AST_NODES}-node ceiling)" + ) + if not isinstance(node, _ALLOWED_NODES): + raise UnsafeEvaluatorSource( + f"{field_name} contains disallowed syntax: {type(node).__name__}" + ) + # Defense in depth: a literal `10 ** 20` (or worse, `2 ** (10**8)`) builds a + # giant int — a memory bomb — at compile-time-visible size. Require Pow's + # exponent to be a small non-negative integer constant. Runtime-sized bombs + # still exist and are caught by the fork sandbox, not here. + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Pow): + exponent = node.right + if not ( + isinstance(exponent, ast.Constant) + and isinstance(exponent.value, int) + and not isinstance(exponent.value, bool) + and 0 <= exponent.value <= MAX_POW_EXPONENT + ): + raise UnsafeEvaluatorSource( + f"{field_name} exponent must be an integer constant " + f"in 0..{MAX_POW_EXPONENT}" + ) + if isinstance(node, ast.Attribute): + if node.attr.startswith("_"): + raise UnsafeEvaluatorSource( + f"{field_name} may not access private or dunder attributes" + ) + if node.attr not in _ALLOWED_ATTRS: + raise UnsafeEvaluatorSource( + f"{field_name} may not access attribute '{node.attr}'" + ) + if node.attr in _METHOD_ATTRS and node not in called_method_nodes: + raise UnsafeEvaluatorSource( + f"{field_name} may reference method '{node.attr}' only to call it; " + "a bare bound method leaks a heap address when stringified" + ) + if isinstance(node, ast.Name) and node.id.startswith("_"): + raise UnsafeEvaluatorSource(f"{field_name} may not access private names") + return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) + + +# CPython's default object repr — `<... at 0x7f...>` — embeds a live heap address +# (an ASLR/memory-layout disclosure). The PRIMARY defense is at compile time: a bound +# method (the only reachable object with such a repr — the result and transcript types +# are all frozen, pointer-free dataclasses) can no longer be referenced as a value +# (`_METHOD_ATTRS` must be called), so no reachable value carries a pointer repr to +# begin with. This output-boundary scan is DEFENSE IN DEPTH. It matches the "... at +# 0xADDR" tail every default repr shares, plus a bare `0x`+hex run — deliberately +# WITHOUT the leading `<`, so reshaping the wrapper (e.g. `str(x).replace("<","")`, +# the way the compile-time hole was originally bypassed) cannot strip the match. +_OBJECT_REPR = re.compile(r" at 0x[0-9a-fA-F]+|0x[0-9a-fA-F]{6,}") + + +def _forbid_object_reprs(field_name: str, value: Any) -> Any: + if _OBJECT_REPR.search(repr(value)): + raise UnsafeEvaluatorSource( + f"{field_name} result may not embed a runtime object repr" + ) + return value + + +def _raw_eval(source: str, kind: str) -> Callable[[Any], Any]: + """Compile server-authored source and return a function that evaluates it and + validates the result. + + Runs INSIDE the sandbox subprocess (see `_sandbox_runner`) — there is no + isolation here. `compile_condition`/`compile_evaluator` have already validated + the AST in the parent; this recompiles as defense in depth so a subprocess + can never eval source the parent has not vetted. + """ + if kind == "condition": + code = _compile( + source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 + if not isinstance(value, (bool, ConditionResult)): + raise TypeError("condition_source must return bool or ConditionResult") + return _forbid_object_reprs("condition_source", value) + + else: + code = _compile( + source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 + if not isinstance(value, EvalResult): + raise TypeError("evaluator_source must return EvalResult") + return _forbid_object_reprs("evaluator_source", value) + + return run + + +def compile_condition( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, +) -> Callable[[Any], bool | ConditionResult]: + # Validate the AST in THIS (parent) process so unsafe/malformed source is + # rejected up front, before any subprocess is spawned. + _compile(source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) + + def condition(session: Any) -> bool | ConditionResult: + # Managed conditions are sandboxed like evaluators — `sum(range(10**10)) > 0` + # in a condition would otherwise block the worker with NO timeout at all. + return _run_sandboxed( + "condition", + source, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + ) + + return condition + + +def compile_evaluator( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, + eval_key: str | None = None, +) -> Callable[[Any], EvalResult]: + _compile(source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) + + def evaluate(session: Any) -> EvalResult: + # The kernel-enforced boundary: this runs in a fork+exec'd subprocess with + # hard CPU/memory/wall-clock limits and a bounded result, killed if it + # exceeds them, so a server-authored compute or result bomb cannot exhaust + # the worker (SEC-001). `eval_key` lets the child validate the result's + # 25-item limit before it crosses back. + return _run_sandboxed( + "evaluator", + source, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + eval_key=eval_key, + ) + + return evaluate diff --git a/sdk/python/tests/fixtures/evaluator_v2/README.md b/sdk/python/tests/fixtures/evaluator_v2/README.md new file mode 100644 index 00000000..e57c93bd --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/README.md @@ -0,0 +1,28 @@ +# Evaluator v2 contract fixtures + +`contract.json` is the Checkpoint 0 wire contract shared by the Rust server and +the zero-dependency Python SDK. The matching copy lives at +`server/tests/fixtures/evaluator_v2/contract.json` in the `agenteye` repository. +Change both copies together. + +Contract rules: + +- The only accepted protocol major is the exact string `"2"`. Unsupported + majors return `426 unsupported_protocol_version`. +- Unknown JSON fields are ignored so either side may add optional fields within + major version 2. Removing, renaming, or changing the meaning of a field needs + a new major version. +- Worker payloads never carry authoritative tenant or evaluator-instance + identity. The server derives those from the credential and leased record. +- `lease_generation` is the fencing token. `409 lease_lost` is terminal for the + affected local execution; the SDK must stop heartbeating or submitting it. +- `submission_id` is an idempotency key. Replaying identical content succeeds; + reusing it for different content returns `409 submission_conflict`. +- Transcript overflow is terminal in v2 (`413 transcript_too_large`); the server + never silently truncates the evaluated input. +- Only errors marked `retryable` may be retried automatically. HTTP method alone + is not enough to decide whether a protocol operation is safe to replay. + +The timing and payload limits in the fixture are normative defaults. A register +response may lower the worker's effective concurrency, heartbeat interval, or +lease duration, but may not raise a client-side payload bound. diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json new file mode 100644 index 00000000..afe0d4ea --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -0,0 +1,251 @@ +{ + "fixture_revision": "evaluator-v2-2026-08-28.3", + "protocol": { + "supported_major_versions": ["2"], + "transcript_schema_version": "2", + "result_schema_version": "2" + }, + "http": { + "register": "/v1/evaluator/workers/register", + "claim": "/v1/evaluator/assignments/claim", + "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "definitions": "/v1/evaluator/assignments/{assignment_id}/definitions", + "plan": "/v1/evaluator/assignments/{assignment_id}/plan", + "heartbeat": "/v1/evaluator/runs/heartbeat", + "result": "/v1/evaluator/runs/{evaluation_run_id}/result", + "worker_id_header": "X-FailproofAI-Worker-Id", + "lease_generation_header": "X-FailproofAI-Lease-Generation" + }, + "timing": { + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "max_attempts": 5 + }, + "limits": { + "max_catalog_definitions": 100, + "max_claim_capacity": 32, + "max_transcript_bytes": 26214400, + "max_results_per_run": 25, + "max_eval_key_bytes": 128, + "max_display_name_bytes": 128, + "max_version_bytes": 128, + "max_worker_id_bytes": 128, + "max_label_bytes": 64, + "max_labels_per_result": 20, + "max_summary_bytes": 4096, + "max_reasoning_bytes": 16384, + "max_unit_bytes": 64, + "max_display_value_bytes": 256, + "max_description_bytes": 1000, + "max_error_code_bytes": 64, + "max_error_message_bytes": 4096 + }, + "errors": { + "invalid_credentials": {"http_status": 401, "retryable": false}, + "instance_disabled": {"http_status": 403, "retryable": false}, + "assignment_not_found": {"http_status": 404, "retryable": false}, + "run_not_found": {"http_status": 404, "retryable": false}, + "catalog_mismatch": {"http_status": 409, "retryable": false}, + "lease_lost": {"http_status": 409, "retryable": false}, + "plan_conflict": {"http_status": 409, "retryable": false}, + "submission_conflict": {"http_status": 409, "retryable": false}, + "retry_budget_exhausted": {"http_status": 409, "retryable": false}, + "transcript_too_large": {"http_status": 413, "retryable": false}, + "invalid_request": {"http_status": 422, "retryable": false}, + "invalid_catalog": {"http_status": 422, "retryable": false}, + "incomplete_plan": {"http_status": 422, "retryable": false}, + "unsupported_protocol_version": {"http_status": 426, "retryable": false}, + "internal_error": {"http_status": 500, "retryable": true} + }, + "samples": { + "register_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "sdk_version": "0.0.1b2", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "max_concurrency": 4, + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"] + } + ] + }, + "register_response": { + "protocol_version": "2", + "evaluator_instance_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23101", + "evaluator_kind": "customer", + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "claim_limit": 4, + "disabled_definitions": [] + }, + "claim_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "capacity": 2 + }, + "claim_response": { + "protocol_version": "2", + "assignments": [ + { + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "lease_generation": 3, + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "trigger_reason": "agent_end", + "event_count": 42, + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript", + "definitions_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/definitions" + } + ] + }, + "definitions_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"], + "execution_mode": "python", + "condition_source": null, + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 + } + ] + }, + "transcript_response": { + "schema_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "started_at": "2026-08-28T11:58:00.000000Z", + "ended_at": "2026-08-28T12:00:00.000000Z", + "event_count": 2, + "events": [ + { + "id": "evt-tool-1", + "ts": "2026-08-28T11:59:00.000000Z", + "event_type": "tool_use", + "payload": {"tool_name": "search"} + }, + { + "id": "evt-end-1", + "ts": "2026-08-28T12:00:00.000000Z", + "event_type": "agent_end", + "payload": {"summary": "Done"} + } + ] + }, + "plan_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "selected": [ + {"eval_key": "tool_efficiency", "eval_version": "1.2.0"} + ], + "skipped": [ + { + "eval_key": "answer_groundedness", + "eval_version": "2.1.0", + "reason_code": "no_retrieval_events" + } + ] + }, + "plan_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "assignment_status": "planned", + "idempotent_replay": false, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "eval_key": "tool_efficiency", + "eval_version": "1.2.0", + "execution_mode": "python", + "evaluator_source": "EvalResult(score=Score(1.0))", + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 + } + ] + }, + "heartbeat_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "state": "running", + "progress": 0.5 + } + ] + }, + "heartbeat_response": { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": ["018f47a8-7c1d-7e21-a22a-79f7a4d23103"] + }, + "result_request": { + "protocol_version": "2", + "result_schema_version": "2", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "status": "succeeded", + "started_at": "2026-08-28T12:00:10.000000Z", + "finished_at": "2026-08-28T12:00:10.812000Z", + "duration_ms": 812, + "summary": "Used a compact tool set without retries.", + "results": [ + { + "result_key": "tool_efficiency", + "result_kind": "score", + "numeric_value": 0.92, + "bool_value": true, + "text_value": null, + "unit": "ratio", + "display_value": "92%", + "description": "Distinct tools divided by total tool calls", + "reasoning": "3 distinct tools across 3 calls", + "labels": ["tools", "deterministic"] + } + ], + "error_code": null, + "error_message": null + }, + "result_response": { + "protocol_version": "2", + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "status": "committed", + "idempotent_replay": false, + "result_count": 1, + "result_checksum": "sha256:8cbd34f2d95d" + }, + "error_response": { + "protocol_version": "2", + "error": { + "code": "lease_lost", + "message": "The assignment lease is no longer owned by this worker.", + "retryable": false, + "request_id": "req-018f47a8" + } + } + } +} diff --git a/sdk/python/tests/test_evaluator_authoring.py b/sdk/python/tests/test_evaluator_authoring.py new file mode 100644 index 00000000..578b1220 --- /dev/null +++ b/sdk/python/tests/test_evaluator_authoring.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +import math + +import pytest + +from failproofai_sdk.evaluator import ( + Assertion, + EvalResult, + Evaluator, + Metric, + ResultKind, + Score, +) + + +def test_catalog_is_stable_across_registration_order(): + first = Evaluator(name="acme", version="2026.08.1") + second = Evaluator(name="acme", version="2026.08.1") + + @first.eval("zeta_check", version="1", labels=["z", "a"]) + def first_zeta(session): + return EvalResult(score=Score(1)) + + @first.eval("alpha_check", version="1") + def first_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("alpha_check", version="1") + def second_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("zeta_check", version="1", labels=["a", "z"]) + def second_zeta(session): + return EvalResult(score=Score(1)) + + assert first.catalog_revision == second.catalog_revision + assert [item.eval_key for item in first.catalog()] == ["alpha_check", "zeta_check"] + + +def test_duplicate_eval_keys_are_rejected_even_when_versions_differ(): + evaluator = Evaluator(name="acme", version="1") + + @evaluator.eval("quality", version="1") + def quality_v1(session): + return EvalResult(score=Score(1)) + + with pytest.raises(ValueError, match="duplicate eval key"): + + @evaluator.eval("quality", version="2") + def quality_v2(session): + return EvalResult(score=Score(1)) + + +@pytest.mark.parametrize("value", [-0.01, 1.01, math.nan, math.inf]) +def test_scores_are_finite_ratios(value): + with pytest.raises(ValueError): + Score(value) + + +def test_result_presentation_fields_are_bounded_before_networking(): + with pytest.raises(ValueError, match="unit is 65 bytes"): + Metric(1, unit="u" * 65) + with pytest.raises(ValueError, match="display value is 257 bytes"): + Score(1, display_value="x" * 257) + with pytest.raises(ValueError, match="description is 1001 bytes"): + Assertion(True, description="x" * 1001) + + +def test_eval_result_expands_to_typed_long_form_rows(): + result = EvalResult( + score=Score(0.75, passed=True, unit="ratio"), + metrics={"call_count": Metric(4, unit="calls")}, + assertions={"had_output": Assertion(True)}, + reasoning="Three useful calls out of four.", + labels=("tools",), + ) + + items = result.result_items("tool_efficiency") + assert [item.result_kind for item in items] == [ + ResultKind.SCORE, + ResultKind.METRIC, + ResultKind.ASSERTION, + ] + assert items[0].reasoning == "Three useful calls out of four." + assert items[1].numeric_value == 4 + assert items[2].bool_value is True + + +def test_empty_eval_result_is_rejected_when_serialized(): + with pytest.raises(ValueError, match="must contain"): + EvalResult().result_items("quality") + + +def test_result_keys_must_be_unique_across_kinds(): + result = EvalResult(score=Score(1), metrics={"quality": 1}) + with pytest.raises(ValueError, match="result keys must be unique"): + result.result_items("quality") + + +def test_sync_and_async_functions_share_one_call_path(): + async def async_eval(session): + return EvalResult(score=Score(1)) + + def sync_eval(session): + return EvalResult(score=Score(0.5)) + + async def exercise(): + sync_result = await Evaluator.call(sync_eval, None) + async_result = await Evaluator.call(async_eval, None) + return sync_result, async_result + + sync_result, async_result = asyncio.run(exercise()) + assert sync_result.score.value == 0.5 + assert async_result.score.value == 1 + + +def test_keys_are_machine_safe_and_versions_are_explicit(): + evaluator = Evaluator(name="acme", version="1") + with pytest.raises(ValueError, match="must match"): + evaluator.eval("Not Safe", version="1") + with pytest.raises(ValueError, match="must not be empty"): + evaluator.eval("safe", version="") diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py new file mode 100644 index 00000000..d74c60dc --- /dev/null +++ b/sdk/python/tests/test_evaluator_client.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError, URLError + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvaluatorAPIError, + EvaluatorClient, + ResultRequest, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class Response: + def __init__(self, body): + self.body = json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def read(self, amount): + return self.body[:amount] + + +class RawResponse(Response): + def __init__(self, body: bytes): + self.body = body + + +def test_claim_sends_bearer_auth_and_does_not_retry(): + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + raise URLError("offline") + + client = EvaluatorClient( + base_url="https://cloud.example/api/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + with pytest.raises(EvaluatorAPIError, match="transport_error"): + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + + assert len(calls) == 1 + request, timeout = calls[0] + assert request.full_url == "https://cloud.example/v1/evaluator/assignments/claim" + assert request.get_header("Authorization") == "Bearer secret" + assert timeout == 30 + + +def test_idempotent_result_submission_retries_transport_failure(): + samples = _samples() + calls = 0 + + def opener(request, timeout): + nonlocal calls + calls += 1 + if calls == 1: + raise URLError("reset") + return Response(samples["result_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + response = client.submit_result( + samples["result_response"]["evaluation_run_id"], + ResultRequest.from_wire(samples["result_request"]), + ) + assert response.status == "committed" + assert calls == 2 + + +def test_transcript_url_cannot_exfiltrate_the_worker_credential(): + sample = _samples()["claim_response"]["assignments"][0] + assignment = Assignment.from_wire( + {**sample, "transcript_url": "https://evil.test/read"} + ) + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda *_args, **_kwargs: pytest.fail("network must not be reached"), + ) + with pytest.raises(EvaluatorAPIError, match="outside the configured API origin"): + client.transcript(assignment, worker_id="worker") + + +def test_machine_error_envelope_controls_retryability(): + body = json.dumps(_samples()["error_response"]).encode() + + def opener(request, timeout): + raise HTTPError(request.full_url, 409, "Conflict", {}, io.BytesIO(body)) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == "lease_lost" + assert caught.value.status == 409 + assert caught.value.retryable is False + assert caught.value.request_id == "req-018f47a8" + + +@pytest.mark.parametrize( + ("body", "code"), + [ + (b"not-json", "invalid_response"), + (b"[]", "invalid_response"), + ( + b"{" + b'"padding":"' + b"x" * (2 * 1024 * 1024) + b'"}', + "response_too_large", + ), + ], +) +def test_malformed_or_oversized_server_responses_fail_closed(body, code): + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda request, timeout: RawResponse(body), + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == code + assert caught.value.retryable is False + + +def test_transcript_identity_is_sent_as_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["transcript_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + client.transcript(assignment, worker_id="worker-7") + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_definitions_use_the_server_supplied_path_and_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["definitions_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + response = client.definitions(assignment, worker_id="worker-7") + + assert response.assignment_id == assignment.assignment_id + assert response.definitions[0].execution_mode.value == "python" + assert captured.full_url.endswith(assignment.definitions_url) + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_constructor_rejects_unsafe_or_incomplete_configuration(): + with pytest.raises(ValueError, match="absolute"): + EvaluatorClient(base_url="localhost:8080", credential="secret") + with pytest.raises(ValueError, match="credential"): + EvaluatorClient(base_url="https://cloud.example", credential="") + with pytest.raises(ValueError, match="control characters"): + EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") + + +def test_private_cluster_http_requires_an_explicit_opt_in(): + with pytest.raises(ValueError, match="must use https"): + EvaluatorClient(base_url="http://server:8080", credential="secret") + EvaluatorClient( + base_url="http://server:8080", + credential="secret", + allow_insecure_http=True, + ) + + +def test_protocol_redirect_does_not_forward_the_bearer_credential(): + exfiltration_attempts = [] + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + exfiltration_attempts.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): + return + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + sink_thread = threading.Thread(target=sink.serve_forever, daemon=True) + sink_thread.start() + + location = f"http://127.0.0.1:{sink.server_address[1]}/steal" + + class Redirector(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(307) + self.send_header("Location", location) + self.end_headers() + + def log_message(self, format, *args): + return + + redirector = ThreadingHTTPServer(("127.0.0.1", 0), Redirector) + redirector_thread = threading.Thread(target=redirector.serve_forever, daemon=True) + redirector_thread.start() + try: + client = EvaluatorClient( + base_url=f"http://127.0.0.1:{redirector.server_address[1]}", + credential="must-not-leak", + max_retries=0, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 0)) + assert caught.value.status == 307 + assert exfiltration_attempts == [] + finally: + redirector.shutdown() + redirector.server_close() + redirector_thread.join(timeout=5) + sink.shutdown() + sink.server_close() + sink_thread.join(timeout=5) diff --git a/sdk/python/tests/test_evaluator_example.py b/sdk/python/tests/test_evaluator_example.py new file mode 100644 index 00000000..e85a545e --- /dev/null +++ b/sdk/python/tests/test_evaluator_example.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ConditionResult + + +def _example_module(): + path = Path(__file__).parents[1] / "examples" / "evaluator_worker.py" + spec = importlib.util.spec_from_file_location("evaluator_worker_example", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_example_registers_deterministic_and_async_evals(monkeypatch): + monkeypatch.delenv("EXAMPLE_JUDGE_URL", raising=False) + module = _example_module() + definitions = {item.eval_key: item for item in module.app.definitions} + assert set(definitions) == {"answer_relevance", "tool_efficiency"} + assert definitions["answer_relevance"].eval_version == "judge-api-v1" + + skipped = definitions["answer_relevance"].condition(None) + assert skipped == ConditionResult(False, "judge_not_configured") + + +def test_example_rejects_non_http_judge_urls(monkeypatch): + module = _example_module() + monkeypatch.setenv("EXAMPLE_JUDGE_URL", "file:///etc/passwd") + with pytest.raises(ValueError, match="absolute http"): + module._call_judge("question", "answer") diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py new file mode 100644 index 00000000..42591bb1 --- /dev/null +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -0,0 +1,636 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit +from uuid import UUID, uuid5 + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvalResult, + EvalSelection, + Evaluator, + EvaluatorAPIError, + EvaluatorClient, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + ResultItem, + ResultKind, + ResultRequest, + Score, + TerminalRunStatus, + WorkerConfig, + WorkerRuntime, +) + +_NAMESPACE = UUID("4d592d9c-aed4-4f07-9b2d-e14963399df6") + + +class ProtocolState: + def __init__(self) -> None: + self.lock = threading.Lock() + self.base_url = "" + self.instances = { + "customer-a-token": ("instance-customer-a", "customer", "org-a"), + "customer-b-token": ("instance-customer-b", "customer", "org-b"), + "managed-token": ("instance-managed", "managed", None), + } + self.registrations: dict[str, dict] = {} + self.assignments: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + self.result_attempts = 0 + self.result_commits = 0 + self.last_result_body: dict | None = None + self.drop_first_result_response = False + + def add_assignment(self, name: str, *, token: str, org: str) -> str: + assignment_id = str(uuid5(_NAMESPACE, name)) + self.assignments[assignment_id] = { + "token": token, + "org": org, + "status": "available", + "worker_id": None, + "lease_generation": 0, + "expired": False, + "session_id": f"session-{name}", + "session_revision_id": f"revision-{name}", + } + return assignment_id + + def expire(self, assignment_id: str) -> None: + with self.lock: + self.assignments[assignment_id]["expired"] = True + + +class ProtocolServer: + def __init__(self, state: ProtocolState) -> None: + self.state = state + handler = _handler_for(state) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.server.daemon_threads = True + state.base_url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> ProtocolServer: # noqa: PYI034 - Python 3.10 lacks Self + self.thread.start() + return self + + def __exit__(self, *_args) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +def _handler_for(state: ProtocolState): + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + body = self._body() + path = urlsplit(self.path).path + if path == "/v1/evaluator/workers/register": + state.registrations[token] = body + instance_id, kind, _org = state.instances[token] + self._json( + 200, + { + "protocol_version": "2", + "evaluator_instance_id": instance_id, + "evaluator_kind": kind, + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "claim_limit": body["max_concurrency"], + "disabled_definitions": [], + }, + ) + return + if path == "/v1/evaluator/assignments/claim": + self._claim(token, body) + return + if path.endswith("/plan"): + self._plan(token, path.split("/")[-2], body) + return + if path == "/v1/evaluator/runs/heartbeat": + self._heartbeat(token, body) + return + if path.endswith("/result"): + self._result(token, path.split("/")[-2], body) + return + self._error(404, "assignment_not_found", False) + + def do_GET(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + path = urlsplit(self.path).path + if path.endswith("/transcript"): + self._transcript(token, path.split("/")[-2]) + return + self._error(404, "assignment_not_found", False) + + def _claim(self, token: str, body: dict) -> None: + claimed = [] + with state.lock: + for assignment_id, item in state.assignments.items(): + if len(claimed) >= body["capacity"]: + break + if item["token"] != token: + continue + if item["status"] in {"leased", "planned"} and not item["expired"]: + continue + if item["status"] not in {"available", "leased", "planned"}: + continue + item["status"] = "leased" + item["expired"] = False + item["worker_id"] = body["worker_id"] + item["lease_generation"] += 1 + claimed.append(self._assignment_wire(assignment_id, item)) + self._json(200, {"protocol_version": "2", "assignments": claimed}) + + def _transcript(self, token: str, assignment_id: str) -> None: + item = self._leased_assignment( + token, + assignment_id, + self.headers.get("X-FailproofAI-Worker-Id"), + self.headers.get("X-FailproofAI-Lease-Generation"), + ) + if item is None: + return + self._json( + 200, + { + "schema_version": "2", + "assignment_id": assignment_id, + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "started_at": "2026-08-28T12:00:00.000000Z", + "ended_at": "2026-08-28T12:00:01.000000Z", + "event_count": 1, + "events": [ + { + "id": "event-1", + "ts": "2026-08-28T12:00:00.500000Z", + "event_type": "model_response", + "payload": {"content": "done"}, + } + ], + }, + ) + + def _plan(self, token: str, assignment_id: str, body: dict) -> None: + item = self._leased_assignment( + token, assignment_id, body["worker_id"], body["lease_generation"] + ) + if item is None: + return + runs = [] + with state.lock: + for selected in body["selected"]: + run_id = str( + uuid5( + _NAMESPACE, + f"{assignment_id}:{selected['eval_key']}:{selected['eval_version']}", + ) + ) + run = state.runs.setdefault( + run_id, + { + "token": token, + "assignment_id": assignment_id, + "worker_id": body["worker_id"], + "lease_generation": body["lease_generation"], + "submission_id": None, + "checksum": None, + }, + ) + if run["submission_id"] is None: + run["worker_id"] = body["worker_id"] + run["lease_generation"] = body["lease_generation"] + runs.append( + { + "evaluation_run_id": run_id, + "execution_mode": "local", + **selected, + } + ) + item["status"] = "planned" if runs else "skipped" + self._json( + 200, + { + "protocol_version": "2", + "assignment_id": assignment_id, + "assignment_status": item["status"], + "runs": runs, + }, + ) + + def _heartbeat(self, token: str, body: dict) -> None: + accepted = [] + with state.lock: + for requested in body["runs"]: + run = state.runs.get(requested["evaluation_run_id"]) + assignment = ( + state.assignments.get(run["assignment_id"]) + if run is not None + else None + ) + if ( + run is not None + and assignment is not None + and run["token"] == token + and run["worker_id"] == body["worker_id"] + and run["lease_generation"] == body["lease_generation"] + and assignment["worker_id"] == body["worker_id"] + and assignment["lease_generation"] == body["lease_generation"] + and not assignment["expired"] + ): + accepted.append(requested["evaluation_run_id"]) + if not accepted: + self._error(409, "lease_lost", False) + return + self._json( + 200, + { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": accepted, + }, + ) + + def _result(self, token: str, run_id: str, body: dict) -> None: + with state.lock: + run = state.runs.get(run_id) + if run is None or run["token"] != token: + self._error(404, "run_not_found", False) + return + if ( + run["worker_id"] != body["worker_id"] + or run["lease_generation"] != body["lease_generation"] + ): + self._error(409, "lease_lost", False) + return + checksum = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if run["submission_id"] is not None: + if ( + run["submission_id"] != body["submission_id"] + or run["checksum"] != checksum + ): + self._error(409, "submission_conflict", False) + return + replay = True + else: + run["submission_id"] = body["submission_id"] + run["checksum"] = checksum + state.result_commits += 1 + state.last_result_body = body + replay = False + state.result_attempts += 1 + drop = state.drop_first_result_response and state.result_attempts == 1 + if drop: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self._json( + 200, + { + "protocol_version": "2", + "evaluation_run_id": run_id, + "submission_id": body["submission_id"], + "status": "committed", + "idempotent_replay": replay, + "result_count": len(body["results"]), + "result_checksum": checksum, + }, + ) + + def _leased_assignment( + self, + token: str, + assignment_id: str, + worker_id: str | None, + generation: str | int | None, + ) -> dict | None: + with state.lock: + item = state.assignments.get(assignment_id) + if item is None or item["token"] != token: + self._error(404, "assignment_not_found", False) + return None + try: + generation = int(generation) if generation is not None else None + except ValueError: + generation = None + if ( + item["status"] != "leased" + or item["expired"] + or item["worker_id"] != worker_id + or item["lease_generation"] != generation + ): + self._error(409, "lease_lost", False) + return None + return item + + def _assignment_wire(self, assignment_id: str, item: dict) -> dict: + return { + "assignment_id": assignment_id, + "lease_generation": item["lease_generation"], + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "trigger_reason": "agent_end", + "event_count": 1, + "transcript_url": ( + f"{state.base_url}/v1/evaluator/assignments/" + f"{assignment_id}/transcript" + ), + } + + def _token(self) -> str | None: + value = self.headers.get("Authorization", "") + return ( + value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + ) + + def _body(self) -> dict: + size = int(self.headers.get("Content-Length", "0")) + return json.loads(self.rfile.read(size)) + + def _error(self, status: int, code: str, retryable: bool) -> None: + self._json( + status, + { + "protocol_version": "2", + "error": { + "code": code, + "message": code.replace("_", " "), + "retryable": retryable, + "request_id": "request-e2e", + }, + }, + ) + + def _json(self, status: int, value: dict) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: + return + + return Handler + + +def _client(state: ProtocolState, token: str) -> EvaluatorClient: + return EvaluatorClient( + base_url=state.base_url, + credential=token, + max_retries=2, + sleeper=lambda _seconds: None, + ) + + +def _claim(client: EvaluatorClient, worker_id: str): + return client.claim( + ClaimRequest( + worker_id=worker_id, + catalog_revision="sha256:" + "a" * 64, + capacity=1, + ) + ) + + +def test_real_http_worker_survives_lost_result_response_without_duplicate_commit(): + state = ProtocolState() + state.add_assignment("runtime", token="customer-a-token", org="org-a") + state.drop_first_result_response = True + evaluator = Evaluator(name="e2e", version="1") + + @evaluator.eval("completion_present", version="1") + def completion_present(session): + return EvalResult(score=Score(float(bool(session.events)))) + + with ProtocolServer(state): + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url=state.base_url, + credential="customer-a-token", + worker_id="worker-a", + ), + client=_client(state, "customer-a-token"), + ) + + async def run(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(run()) == 1 + + run_id = next(iter(state.runs)) + committed = ResultRequest.from_wire(state.last_result_body) + replay = runtime.client.submit_result(run_id, committed) + assert replay.idempotent_replay is True + + with pytest.raises(EvaluatorAPIError) as caught: + runtime.client.submit_result( + run_id, + replace(committed, summary="different content"), + ) + assert caught.value.code == "submission_conflict" + + assert state.result_attempts == 3 + assert state.result_commits == 1 + assert len(state.runs) == 1 + assert next(iter(state.runs.values()))["submission_id"] is not None + assert runtime.metrics()["runs_succeeded"] == 1 + + +def test_two_workers_racing_receive_one_unique_lease(): + state = ProtocolState() + assignment_id = state.add_assignment("race", token="customer-a-token", org="org-a") + with ProtocolServer(state): + first = _client(state, "customer-a-token") + second = _client(state, "customer-a-token") + with ThreadPoolExecutor(max_workers=2) as executor: + responses = list( + executor.map( + lambda pair: _claim(*pair), + [(first, "worker-a"), (second, "worker-b")], + ) + ) + + claimed = [item for response in responses for item in response.assignments] + assert [item.assignment_id for item in claimed] == [assignment_id] + assert state.assignments[assignment_id]["lease_generation"] == 1 + + +def test_expired_lease_is_reclaimed_and_stale_worker_is_fenced(): + state = ProtocolState() + assignment_id = state.add_assignment( + "reclaim", token="customer-a-token", org="org-a" + ) + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + + assert second.lease_generation == first.lease_generation + 1 + with pytest.raises(EvaluatorAPIError) as caught: + client.transcript(first, worker_id="worker-a") + assert caught.value.code == "lease_lost" + assert caught.value.retryable is False + + with pytest.raises(EvaluatorAPIError) as caught: + client.heartbeat( + HeartbeatRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + runs=(HeartbeatRun(plan.runs[0].evaluation_run_id, "running"),), + ) + ) + assert caught.value.code == "lease_lost" + + +def test_replacement_worker_finishes_after_forced_worker_loss(): + state = ProtocolState() + assignment_id = state.add_assignment( + "forced-worker-loss", token="customer-a-token", org="org-a" + ) + result_sample = ResultRequest( + submission_id=str(uuid5(_NAMESPACE, "forced-worker-loss-result")), + worker_id="worker-a", + lease_generation=1, + status=TerminalRunStatus.SUCCEEDED, + started_at="2026-08-28T12:00:10.000000Z", + finished_at="2026-08-28T12:00:10.100000Z", + duration_ms=100, + summary="Replacement worker completed the evaluation.", + results=( + ResultItem( + result_key="quality", + result_kind=ResultKind.SCORE, + numeric_value=1.0, + ), + ), + error_code=None, + error_message=None, + ) + + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + first_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + + # Worker A disappears after planning. Its lease expires and worker B + # reclaims the same logical assignment and deterministic run. + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + second_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-b", + lease_generation=second.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + assert ( + second_plan.runs[0].evaluation_run_id + == first_plan.runs[0].evaluation_run_id + ) + + stale_result = replace( + result_sample, + worker_id="worker-a", + lease_generation=first.lease_generation, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.submit_result(first_plan.runs[0].evaluation_run_id, stale_result) + assert caught.value.code == "lease_lost" + + replacement_result = replace( + result_sample, + worker_id="worker-b", + lease_generation=second.lease_generation, + ) + committed = client.submit_result( + second_plan.runs[0].evaluation_run_id, replacement_result + ) + assert committed.status == "committed" + assert committed.idempotent_replay is False + + assert state.result_attempts == 1 + assert state.result_commits == 1 + + +def test_customer_tenants_are_isolated_and_managed_worker_coexists(): + state = ProtocolState() + customer_id = state.add_assignment( + "customer", token="customer-a-token", org="org-a" + ) + managed_id = state.add_assignment("managed", token="managed-token", org="org-b") + with ProtocolServer(state): + customer_a = _client(state, "customer-a-token") + customer_b = _client(state, "customer-b-token") + managed = _client(state, "managed-token") + + customer_assignment = _claim(customer_a, "worker-a").assignments[0] + assert customer_assignment.assignment_id == customer_id + assert _claim(customer_b, "worker-b").assignments == () + assert ( + _claim(managed, "worker-managed").assignments[0].assignment_id == managed_id + ) + + stolen = Assignment( + assignment_id=customer_assignment.assignment_id, + lease_generation=customer_assignment.lease_generation, + lease_expires_at=customer_assignment.lease_expires_at, + session_id=customer_assignment.session_id, + session_revision_id=customer_assignment.session_revision_id, + agent_id=customer_assignment.agent_id, + environment=customer_assignment.environment, + trigger_reason=customer_assignment.trigger_reason, + event_count=customer_assignment.event_count, + transcript_url=customer_assignment.transcript_url, + ) + with pytest.raises(EvaluatorAPIError) as caught: + customer_b.transcript(stolen, worker_id="worker-a") + assert caught.value.status == 404 + assert caught.value.code == "assignment_not_found" diff --git a/sdk/python/tests/test_evaluator_main.py b/sdk/python/tests/test_evaluator_main.py new file mode 100644 index 00000000..c846abdc --- /dev/null +++ b/sdk/python/tests/test_evaluator_main.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sys + +import pytest + +from failproofai_sdk.evaluator import Evaluator +from failproofai_sdk.evaluator.__main__ import load_evaluator + + +def test_module_loader_defaults_to_app(tmp_path, monkeypatch): + (tmp_path / "my_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "app = Evaluator(name='example', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("my_evals") + finally: + sys.modules.pop("my_evals", None) + assert isinstance(loaded, Evaluator) + assert loaded.name == "example" + + +def test_module_loader_supports_an_explicit_attribute(tmp_path, monkeypatch): + (tmp_path / "custom_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "worker = Evaluator(name='custom', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("custom_evals:worker") + finally: + sys.modules.pop("custom_evals", None) + assert loaded.name == "custom" + + +def test_module_loader_rejects_the_wrong_object_type(tmp_path, monkeypatch): + (tmp_path / "not_evals.py").write_text("app = object()\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + with pytest.raises(TypeError, match="not Evaluator"): + load_evaluator("not_evals") + finally: + sys.modules.pop("not_evals", None) diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py new file mode 100644 index 00000000..e23b2474 --- /dev/null +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimRequest, + ClaimResponse, + DefinitionsResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + UnsupportedProtocolVersion, + protocol, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _contract(): + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("sample", "model"), + [ + ("register_request", RegisterRequest), + ("register_response", RegisterResponse), + ("claim_request", ClaimRequest), + ("claim_response", ClaimResponse), + ("definitions_response", DefinitionsResponse), + ("transcript_response", SessionTranscript), + ("plan_request", PlanRequest), + ("plan_response", PlanResponse), + ("heartbeat_request", HeartbeatRequest), + ("heartbeat_response", HeartbeatResponse), + ("result_request", ResultRequest), + ("result_response", ResultResponse), + ("error_response", ErrorResponse), + ], +) +def test_golden_messages_round_trip(sample, model): + wire = _contract()["samples"][sample] + assert model.from_wire(wire).to_wire() == wire + + +def test_unknown_additive_fields_are_tolerated(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["future_optional_field"] = True + assert ClaimRequest.from_wire(wire).capacity == 2 + + +def test_unsupported_major_version_fails_loudly(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["protocol_version"] = "3" + with pytest.raises( + UnsupportedProtocolVersion, match="supported major version is 2" + ): + ClaimRequest.from_wire(wire) + + +def test_transcript_event_count_is_an_integrity_check(): + wire = dict(_contract()["samples"]["transcript_response"]) + wire["event_count"] = 99 + with pytest.raises(ProtocolError, match="transcript contains 2 events"): + SessionTranscript.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "path", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + ("assignments", 0), + 42, + r"assignments\[0\] must be an object", + ), + ( + "register_response", + RegisterResponse, + ("disabled_definitions",), + [42], + r"disabled_definitions\[0\] must be a string", + ), + ( + "heartbeat_response", + HeartbeatResponse, + ("accepted_run_ids", 0), + None, + r"accepted_run_ids\[0\] must be a string", + ), + ( + "plan_response", + PlanResponse, + ("runs", 0), + "not-an-object", + r"runs\[0\] must be an object", + ), + ( + "result_request", + ResultRequest, + ("results", 0, "labels", 0), + 7, + r"labels\[0\] must be a string", + ), + ( + "register_request", + RegisterRequest, + ("definitions", 0, "result_kind"), + "unknown", + "result_kind must be one of", + ), + ], +) +def test_nested_wire_values_fail_with_protocol_errors( + sample, model, path, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + target = wire + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "field", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + "lease_generation", + 0, + "lease_generation must be greater than zero", + ), + ( + "claim_response", + ClaimResponse, + "event_count", + -1, + "event_count must not be negative", + ), + ( + "result_response", + ResultResponse, + "result_count", + -1, + "result_count must not be negative", + ), + ], +) +def test_server_response_counters_and_generations_are_bounded( + sample, model, field, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + if sample == "claim_response": + wire["assignments"][0][field] = value + else: + wire[field] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +def test_fixture_constants_match_the_sdk_contract(): + contract = _contract() + assert contract["protocol"] == { + "supported_major_versions": [protocol.PROTOCOL_VERSION], + "transcript_schema_version": protocol.TRANSCRIPT_SCHEMA_VERSION, + "result_schema_version": protocol.RESULT_SCHEMA_VERSION, + } + assert contract["http"] == { + "register": protocol.REGISTER_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "definitions": protocol.DEFINITIONS_PATH, + "plan": protocol.PLAN_PATH, + "heartbeat": protocol.HEARTBEAT_PATH, + "result": protocol.RESULT_PATH, + "worker_id_header": protocol.WORKER_ID_HEADER, + "lease_generation_header": protocol.LEASE_GENERATION_HEADER, + } + assert contract["timing"] == { + "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS, + "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS, + "poll_interval_seconds": protocol.DEFAULT_POLL_INTERVAL_SECONDS, + "max_attempts": protocol.MAX_ATTEMPTS, + } + assert contract["limits"] == { + "max_catalog_definitions": protocol.MAX_CATALOG_DEFINITIONS, + "max_claim_capacity": protocol.MAX_CLAIM_CAPACITY, + "max_transcript_bytes": protocol.MAX_TRANSCRIPT_BYTES, + "max_results_per_run": protocol.MAX_RESULTS_PER_RUN, + "max_eval_key_bytes": protocol.MAX_EVAL_KEY_BYTES, + "max_display_name_bytes": protocol.MAX_DISPLAY_NAME_BYTES, + "max_version_bytes": protocol.MAX_VERSION_BYTES, + "max_worker_id_bytes": protocol.MAX_WORKER_ID_BYTES, + "max_label_bytes": protocol.MAX_LABEL_BYTES, + "max_labels_per_result": protocol.MAX_LABELS_PER_RESULT, + "max_summary_bytes": protocol.MAX_SUMMARY_BYTES, + "max_reasoning_bytes": protocol.MAX_REASONING_BYTES, + "max_unit_bytes": protocol.MAX_UNIT_BYTES, + "max_display_value_bytes": protocol.MAX_DISPLAY_VALUE_BYTES, + "max_description_bytes": protocol.MAX_DESCRIPTION_BYTES, + "max_error_code_bytes": protocol.MAX_ERROR_CODE_BYTES, + "max_error_message_bytes": protocol.MAX_ERROR_MESSAGE_BYTES, + } + assert contract["errors"] == protocol.ERROR_SPECS + + +def test_session_helpers_use_the_protocol_event_vocabulary(): + session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"]) + assert session.count("tool_use") == 1 + assert session.events_of_type("agent_end")[0].payload["summary"] == "Done" + + +def test_falsy_or_missing_execution_mode_is_rejected_not_defaulted(): + # F2: a falsy/absent execution_mode was silently coerced to 'local', which + # could run a server-authored ('python') definition down the customer path. + # It must now be a hard protocol error, not a default. + samples = _contract()["samples"] + for bad in ("", None): + defs = json.loads(json.dumps(samples["definitions_response"])) + defs["definitions"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(defs) + plan = json.loads(json.dumps(samples["plan_response"])) + plan["runs"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + PlanResponse.from_wire(plan) + absent = json.loads(json.dumps(samples["definitions_response"])) + del absent["definitions"][0]["execution_mode"] + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(absent) diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py new file mode 100644 index 00000000..dff10869 --- /dev/null +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -0,0 +1,1103 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + AssignmentDefinition, + ClaimResponse, + ConditionResult, + DefinitionsResponse, + EvalResult, + Evaluator, + EvaluatorAPIError, + ExecutionMode, + HeartbeatResponse, + PlannedRun, + PlanResponse, + RegisterResponse, + ResultKind, + Score, + SessionTranscript, + WorkerConfig, + WorkerRuntime, + source_checksum, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class FakeClient: + def __init__(self): + samples = _samples() + self.assignment = ClaimResponse.from_wire( + samples["claim_response"] + ).assignments[0] + self.assignment = replace(self.assignment, definitions_url="") + self.session = SessionTranscript.from_wire(samples["transcript_response"]) + self.register_requests = [] + self.claim_requests = [] + self.plans = [] + self.submissions = [] + self.heartbeats = [] + + def register(self, request): + self.register_requests.append(request) + return RegisterResponse.from_wire(_samples()["register_response"]) + + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=(self.assignment,)) + + def transcript(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return self.session + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned" if request.selected else "skipped", + runs=tuple( + PlannedRun(f"run-{item.eval_key}", item.eval_key, item.eval_version) + for item in request.selected + ), + ) + + def submit_result(self, run_id, request): + self.submissions.append((run_id, request)) + + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=tuple(item.evaluation_run_id for item in request.runs), + ) + + +def _runtime(evaluator, client): + return WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=2, + ), + client=client, + ) + + +def test_managed_definition_is_fetched_verified_and_executed(): + source = "EvalResult(score=Score(0.75, passed=True), summary='hosted')" + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, source), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=source, + source_checksum=source_checksum(None, source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "succeeded" + assert result.summary == "hosted" + assert result.results[0].numeric_value == 0.75 + + +def test_managed_definition_that_fails_to_compile_dead_letters_as_one_failed_run(): + # Unsafe/malformed server-authored source is rejected by the sandbox at + # compile time. That rejection must surface as a single bounded FAILED run, + # NOT as an exception out of assignment setup that crashes the task and + # forces the whole assignment to be reclaimed and retried. + unsafe = ( + 'EvalResult(score=Score(1.0), ' + 'reasoning="{0.__class__}".format(session))' + ) + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, unsafe), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=unsafe, + source_checksum=source_checksum(None, unsafe), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + # Must NOT raise — the poison definition is contained to its own run. + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "failed" + assert result.error_code == "eval_error" + # Nothing derived from the rejected source may be reported. + assert result.results == () + assert result.summary is None + + +def test_managed_condition_governs_even_when_a_local_key_collides(): + # COR-001: `local` is keyed on (eval_key, eval_version) alone, so a managed + # (PYTHON) definition can collide with a local one the worker also registered. + # The server's managed condition must decide applicability — NOT the matching + # local condition. Here the local condition returns True and the managed + # `condition_source` is "False": the definition must be recorded as skipped + # (condition_false) and the managed evaluator source must never run. + source = "EvalResult(score=Score(1.0), summary='should never run')" + + evaluator = Evaluator(name="managed", version="1") + + @evaluator.eval("hosted_quality", version="1", when=lambda session: True) + def hosted_quality(session): # a colliding LOCAL definition, condition True + return EvalResult(score=Score(1.0, passed=True), summary="local") + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + condition_source="False", + source_checksum=source_checksum("False", source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + # The server's managed condition (False) wins over the local one (True): + # recorded as skipped, nothing selected, and no managed run submitted. + assert client.plans[0].selected == () + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("hosted_quality", "condition_false"), + } + assert client.submissions == [] + + +def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): + evaluator = Evaluator(name="parallel", version="1") + lock = threading.Lock() + active = 0 + peak = 0 + + def measured(_session): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.04) + with lock: + active -= 1 + return EvalResult(score=Score(1)) + + for index in range(5): + evaluator.eval( + f"eval_{index}", + version="1", + when=lambda session, index=index: ( + index < 3 if session.session_id == "session-a" else index >= 3 + ), + )(measured) + + class ParallelClient(FakeClient): + def transcript(self, assignment, *, worker_id): + assert worker_id == "worker-test" + return replace( + self.session, + assignment_id=assignment.assignment_id, + session_id=assignment.session_id, + session_revision_id=assignment.session_revision_id, + ) + + client = ParallelClient() + first = replace( + client.assignment, + assignment_id="assignment-a", + session_id="session-a", + session_revision_id="revision-a", + ) + second = replace( + client.assignment, + assignment_id="assignment-b", + session_id="session-b", + session_revision_id="revision-b", + ) + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + await asyncio.gather( + runtime.process_assignment(first), runtime.process_assignment(second) + ) + + asyncio.run(exercise()) + + assert peak == 2 + assert len(client.submissions) == 5 + assert client.heartbeats + + +def test_condition_failures_are_isolated_and_plan_is_declared_first(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("not_applicable", version="1", when=lambda session: False) + def not_applicable(session): + return EvalResult(score=Score(1)) + + def broken_condition(session): + raise RuntimeError("condition exploded") + + @evaluator.eval("broken_condition", version="1", when=broken_condition) + def never_runs(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + assert len(client.plans) == 1 + assert [item.eval_key for item in client.plans[0].selected] == ["selected"] + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("not_applicable", "condition_false"), + ("broken_condition", "condition_error"), + } + assert [run_id for run_id, _ in client.submissions] == ["run-selected"] + + +def test_condition_can_supply_a_stable_skip_reason(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval( + "retrieval_only", + version="1", + when=lambda session: ConditionResult(False, "no_retrieval_events"), + ) + def retrieval_only(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans[0].skipped[0].reason_code == "no_retrieval_events" + + +def test_one_eval_failure_does_not_block_another_result(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("fails", version="1") + def fails(session): + raise RuntimeError("secret details should be bounded") + + @evaluator.eval("succeeds", version="1") + async def succeeds(session): + await asyncio.sleep(0) + return EvalResult(score=Score(0.8), summary="good") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + by_run = {run_id: request for run_id, request in client.submissions} + assert by_run["run-fails"].status.value == "failed" + assert by_run["run-fails"].error_code == "eval_error" + assert by_run["run-fails"].results == () + assert by_run["run-succeeds"].status.value == "succeeded" + assert by_run["run-succeeds"].results[0].result_kind == ResultKind.SCORE + + +def test_timeout_is_submitted_as_a_terminal_run(): + evaluator = Evaluator(name="test", version="1") + cancelled = [] + + @evaluator.eval( + "slow", + version="1", + timeout_seconds=0.01, + on_cancel=lambda session: cancelled.append(session.session_revision_id), + ) + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert cancelled == [client.assignment.session_revision_id] + + +def test_lost_lease_cancels_local_execution(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1") + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + class LeaseLostClient(FakeClient): + def heartbeat(self, request): + raise EvaluatorAPIError( + status=409, + code="lease_lost", + message="gone", + retryable=False, + ) + + client = LeaseLostClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + with pytest.raises(asyncio.CancelledError): + asyncio.run(runtime.process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_partial_heartbeat_acceptance_cancels_only_the_fenced_run(): + evaluator = Evaluator(name="test", version="1") + + class PartialHeartbeatClient(FakeClient): + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=(request.runs[0].evaluation_run_id,), + ) + + client = PartialHeartbeatClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + first = asyncio.create_task(asyncio.sleep(60)) + second = asyncio.create_task(asyncio.sleep(60)) + heartbeat = asyncio.create_task( + runtime._heartbeat( + client.assignment, {"run-first": first, "run-second": second} + ) + ) + while not client.heartbeats: + await asyncio.sleep(0.001) + for _ in range(100): + if second.done(): + break + await asyncio.sleep(0.001) + assert first.done() is False + assert second.cancelled() is True + heartbeat.cancel() + first.cancel() + await asyncio.gather(first, second, heartbeat, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_transcript_revision_must_match_the_claimed_assignment(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + client.session = SessionTranscript.from_wire( + { + **_samples()["transcript_response"], + "session_revision_id": "different-revision", + } + ) + + with pytest.raises(RuntimeError, match="revision does not match"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans == [] + + +def test_server_cannot_add_a_run_when_every_eval_was_skipped(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + class UnexpectedRunClient(FakeClient): + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-injected", "skipped", "1"),), + ) + + client = UnexpectedRunClient() + with pytest.raises(RuntimeError, match="unrequested evaluation run"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_server_plan_must_match_assignment_and_include_each_new_selected_eval(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1") + def selected(session): + return EvalResult(score=Score(1)) + + class WrongAssignmentClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id="another-assignment", + assignment_status="planned", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_assignment = WrongAssignmentClient() + with pytest.raises(RuntimeError, match="different assignment"): + asyncio.run( + _runtime(evaluator, wrong_assignment).process_assignment( + wrong_assignment.assignment + ) + ) + + class WrongStatusClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_status = WrongStatusClient() + with pytest.raises(RuntimeError, match="inconsistent assignment status"): + asyncio.run( + _runtime(evaluator, wrong_status).process_assignment( + wrong_status.assignment + ) + ) + + class OmittedRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + ) + + omitted = OmittedRunClient() + with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): + asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + + class ReplayedPlanClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + idempotent_replay=True, + ) + + replayed = ReplayedPlanClient() + asyncio.run(_runtime(evaluator, replayed).process_assignment(replayed.assignment)) + assert replayed.submissions == [] + + +def test_server_plan_rejects_duplicate_run_ids(): + evaluator = Evaluator(name="test", version="1") + evaluator.eval("first", version="1")(lambda session: EvalResult(score=Score(1))) + evaluator.eval("second", version="1")(lambda session: EvalResult(score=Score(1))) + + class DuplicateRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun("same-run", "first", "1"), + PlannedRun("same-run", "second", "1"), + ), + ) + + client = DuplicateRunClient() + with pytest.raises(RuntimeError, match="duplicate evaluation run id"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_register_advertises_the_deterministic_catalog(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("quality", version="7") + def quality(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + asyncio.run(runtime.register()) + request = client.register_requests[0] + assert request.catalog_revision == evaluator.catalog_revision + assert request.definitions[0].eval_version == "7" + assert runtime._heartbeat_interval == 30 + + +def test_runtime_readiness_tracks_registration_contact_and_shutdown(monkeypatch): + evaluator = Evaluator(name="test", version="1") + runtime = _runtime(evaluator, FakeClient()) + + assert runtime.is_ready() is False + assert runtime.metrics() == {} + + asyncio.run(runtime.register()) + assert runtime.is_ready() is True + assert runtime.metrics() == {"registration_success": 1} + + last_contact = runtime._last_server_contact + assert last_contact is not None + monkeypatch.setattr(time, "monotonic", lambda: last_contact + 121) + assert runtime.is_ready() is False + + monkeypatch.setattr(time, "monotonic", lambda: last_contact) + runtime.stop() + assert runtime.is_ready() is False + + +def test_runtime_metrics_count_claims_conditions_and_outcomes(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + runtime = _runtime(evaluator, FakeClient()) + + async def exercise(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(exercise()) == 1 + assert runtime.metrics() == { + "assignments_claimed": 1, + "conditions_selected": 1, + "conditions_skipped": 1, + "registration_success": 1, + "runs_succeeded": 1, + } + + +def test_runtime_metrics_count_registration_failure(): + evaluator = Evaluator(name="test", version="1") + + class BrokenClient(FakeClient): + def register(self, request): + raise EvaluatorAPIError( + status=503, + code="unavailable", + message="try later", + retryable=True, + ) + + runtime = _runtime(evaluator, BrokenClient()) + with pytest.raises(EvaluatorAPIError): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_invalid_registration_response_does_not_make_runtime_ready(): + evaluator = Evaluator(name="test", version="1") + + class InvalidTimingClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=120, + lease_duration_seconds=120, + poll_interval_seconds=10, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, InvalidTimingClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_lost_claim_response_waits_out_the_lease_before_claiming_again(): + evaluator = Evaluator(name="test", version="1") + + class LostResponseClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=None, + code="transport_error", + message="response lost", + retryable=True, + ) + + runtime = _runtime(evaluator, LostResponseClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [120.0] + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +def test_idle_claim_waits_the_advertised_poll_interval_before_polling_again(): + # Normal short polling: an empty claim returns immediately (no long-poll), so + # the worker sleeps the server-advertised poll_interval_seconds — 10 in the + # fixture register response — instead of hot-looping. The claim request also no + # longer carries a wait_seconds field. + evaluator = Evaluator(name="test", version="1") + + class IdleClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=()) + + runtime = _runtime(evaluator, IdleClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [10.0] + assert len(runtime.client.claim_requests) == 1 + assert not hasattr(runtime.client.claim_requests[0], "wait_seconds") + + +def test_nonretryable_claim_failure_stops_the_worker(): + evaluator = Evaluator(name="test", version="1") + + class RejectedClaimClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=409, + code="catalog_mismatch", + message="register again with the current catalog", + retryable=False, + ) + + runtime = _runtime(evaluator, RejectedClaimClient()) + with pytest.raises(EvaluatorAPIError, match="catalog_mismatch"): + asyncio.run(runtime.run_forever()) + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +@pytest.mark.parametrize( + ("assignments", "message"), + [ + (lambda item: (item, item), "duplicate assignments"), + ( + lambda item: tuple( + replace(item, assignment_id=f"assignment-{index}") for index in range(3) + ), + "more assignments than requested", + ), + ], +) +def test_claim_response_cannot_exceed_capacity_or_repeat_work(assignments, message): + evaluator = Evaluator(name="test", version="1") + + class InvalidClaimClient(FakeClient): + def claim(self, request): + return ClaimResponse(assignments=assignments(self.assignment)) + + runtime = _runtime(evaluator, InvalidClaimClient()) + with pytest.raises(RuntimeError, match=message): + asyncio.run(runtime.run_once()) + assert runtime.metrics() == {} + + +def test_register_applies_server_claim_limit_and_disabled_definitions(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("disabled", version="1") + def disabled(session): + raise AssertionError("disabled eval must not run") + + class RestrictedClient(FakeClient): + def register(self, request): + self.register_requests.append(request) + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=10, + lease_duration_seconds=120, + poll_interval_seconds=10, + claim_limit=1, + disabled_definitions=("disabled",), + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + client = RestrictedClient() + runtime = _runtime(evaluator, client) + + async def exercise(): + await runtime.register() + await runtime.run_once() + + asyncio.run(exercise()) + assert runtime._claim_limit == 1 + assert client.claim_requests[0].capacity == 1 + assert client.plans[0].selected == () + assert client.plans[0].skipped[0].reason_code == "disabled_by_server" + + +def test_worker_config_requires_dedicated_credentials(monkeypatch): + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_URL", raising=False) + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_TOKEN", raising=False) + with pytest.raises(ValueError, match="URL is required"): + WorkerConfig.from_env() + + +def test_register_rejects_non_positive_poll_interval(): + # The worker adopts the server-advertised poll_interval_seconds (normal short + # polling — there is no long-poll wait). A non-positive interval would make the + # claim loop hot-spin, so registration must refuse it. + evaluator = Evaluator(name="test", version="1") + + class ZeroPollClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=30, + lease_duration_seconds=120, + poll_interval_seconds=0, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, ZeroPollClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + + +def test_worker_config_rejects_header_control_characters(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_WORKER_ID", "worker\nforged") + with pytest.raises(ValueError, match="control characters"): + WorkerConfig.from_env() + + +def test_graceful_drain_cancels_work_after_the_configured_deadline(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + drain_timeout_seconds=1, + ), + client=client, + ) + cancelled = False + + async def exercise(): + nonlocal cancelled + + async def active_work(): + nonlocal cancelled + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled = True + raise + + task = asyncio.create_task(active_work()) + runtime._active.add(task) + await asyncio.sleep(0) + runtime.config = WorkerConfig( + server_url=runtime.config.server_url, + credential=runtime.config.credential, + worker_id=runtime.config.worker_id, + drain_timeout_seconds=0, + ) + await runtime.drain() + + asyncio.run(exercise()) + assert cancelled is True + assert runtime._active == set() + + +def test_stop_interrupts_capacity_wait_and_enters_drain(): + runtime = _runtime(Evaluator(name="test", version="1"), FakeClient()) + + async def exercise(): + blocker = asyncio.Event() + work = asyncio.create_task(blocker.wait()) + runtime._active.add(work) + await asyncio.sleep(0) + + runtime.stop() + await asyncio.wait_for(runtime._wait_for_progress(), timeout=0.1) + + assert work.done() is False + work.cancel() + await asyncio.gather(work, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_eval_execution_respects_process_concurrency(): + evaluator = Evaluator(name="test", version="1") + active = 0 + peak = 0 + + async def measured(session): + nonlocal active, peak + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) + active -= 1 + return EvalResult(score=Score(1)) + + evaluator.eval("first", version="1")(measured) + evaluator.eval("second", version="1")(measured) + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=1, + ), + client=client, + ) + asyncio.run(runtime.process_assignment(client.assignment)) + assert peak == 1 + DefinitionsResponse, + ExecutionMode, + + +def test_synchronous_evaluation_timeout_is_counted_as_orphaned(): + # A synchronous evaluator that overruns its timeout cannot be cancelled: the + # runtime submits a terminal timed_out result and records the orphaned thread + # so a hung evaluator is findable. The executor is sized with headroom over + # the concurrency limit so this orphan does not starve live capacity. + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1", timeout_seconds=0.05) + def slow(session): + time.sleep(0.5) + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + try: + asyncio.run(runtime.process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert runtime.metrics().get("sync_evaluations_orphaned") == 1 + assert runtime._eval_executor._max_workers > runtime.config.max_concurrency + finally: + runtime._eval_executor.shutdown(wait=True) + + +def test_conditions_are_skipped_when_the_lease_is_exhausted(): + # With no lease time left before the plan must be submitted, the worker skips + # the condition (without running it) instead of burning the lease and getting + # the plan fenced as lease_lost. + evaluator = Evaluator(name="test", version="1") + ran = [] + + def gate(session): + ran.append(True) + return True + + @evaluator.eval("slow", version="1", when=gate) + def slow(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + # Force an already-exhausted condition-phase deadline. + runtime._condition_phase_deadline = lambda assignment: time.monotonic() + asyncio.run(runtime.process_assignment(client.assignment)) + + assert ran == [], "the condition must not run once the lease is exhausted" + assert client.plans, "a plan must still be submitted" + plan_request = client.plans[-1] + assert not plan_request.selected + reasons = {(s.eval_key, s.reason_code) for s in plan_request.skipped} + assert ("slow", "lease_exhausted") in reasons + assert runtime.metrics().get("conditions_lease_exhausted") == 1 + assert client.submissions == [] + + +def test_condition_phase_deadline_and_budget_are_lease_bounded(): + from datetime import datetime, timedelta, timezone + + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = _runtime(evaluator, client) + runtime._lease_duration = 120 + + # A stale (past) lease_expires_at falls back to the negotiated lease duration, + # so the bound never fires spuriously under clock skew or a replayed fixture. + stale = replace(client.assignment, lease_expires_at="2000-01-01T00:00:00.000000Z") + fallback = runtime._condition_phase_deadline(stale) - time.monotonic() + assert 110 <= fallback <= 125 + + # A future lease is honored. + future_ts = (datetime.now(timezone.utc) + timedelta(seconds=300)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + "Z" + future = replace(client.assignment, lease_expires_at=future_ts) + ahead = runtime._condition_phase_deadline(future) - time.monotonic() + assert 250 <= ahead <= 305 + + # Budget is capped by both the remaining lease and the per-definition timeout. + deadline = time.monotonic() + 100 + assert runtime._condition_budget(deadline, None) == pytest.approx(95, abs=2) + assert runtime._condition_budget(deadline, 10) == pytest.approx(10, abs=0.05) + assert runtime._condition_budget(time.monotonic(), None) < 0 diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py new file mode 100644 index 00000000..fc65bcd5 --- /dev/null +++ b/sdk/python/tests/test_evaluator_source.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import pytest + +from failproofai_sdk.evaluator import EvalResult, Score +from failproofai_sdk.evaluator.protocol import SessionTranscript, TranscriptEvent +from failproofai_sdk.evaluator.source import ( + MAX_AST_NODES, + MAX_EVALUATOR_SOURCE_BYTES, + MAX_POW_EXPONENT, + MAX_SANDBOX_TIMEOUT_SECONDS, + EvaluationSandboxUnavailable, + EvaluationTimeout, + UnsafeEvaluatorSource, + _clamp_budget, + compile_condition, + compile_evaluator, + source_checksum, +) + + +def Session(event_count: int = 3) -> SessionTranscript: + """A real, serializable transcript — managed evals now run in a subprocess and + the transcript crosses the boundary via `to_wire`, so a dummy object won't do. + Each event carries a dict payload (so `events[0].payload.get` is reachable).""" + events = tuple( + TranscriptEvent( + id=f"e{i}", + ts="2026-08-28T12:00:00.000000Z", + event_type="tool_use", + payload={"k": "v", "tool_name": "search"}, + ) + for i in range(event_count) + ) + return SessionTranscript( + assignment_id="a", + session_id="s", + session_revision_id="r", + agent_id="agent", + environment="test", + started_at="2026-08-28T12:00:00.000000Z", + ended_at="2026-08-28T12:00:01.000000Z", + event_count=event_count, + events=events, + ) + + +def test_restricted_expressions_can_evaluate_conditions_and_results(): + assert compile_condition("session.event_count > 0")(Session()) is True + result = compile_evaluator("EvalResult(score=Score(0.75, passed=True))")( + Session() + ) + assert isinstance(result, EvalResult) + assert result.score == Score(0.75, passed=True) + + +@pytest.mark.parametrize( + "source", + [ + "__import__('os').system('id')", + "session.__class__", + "(lambda: 1)()", + "[x for x in ().__class__.__base__.__subclasses__()]", + ], +) +def test_restricted_expressions_reject_escape_primitives(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +@pytest.mark.parametrize( + "source", + [ + # `str.format` / `str.format_map` traverse a format string's field spec + # at the C level, reaching attributes the AST dunder guard never sees. + # These reached real `__builtins__` before the denylist landed. + '"{0.__class__.__init__.__globals__[__builtins__][__import__]}".format(session)', + '"{0.__class__}".format(session)', + 'str.format("{0.__class__}", session)', + '"{a.__class__}".format_map({"a": session})', + # A reasoning string is where a leak would surface — block it there too. + 'EvalResult(score=Score(0.5), reasoning="{0.__class__}".format(session))', + ], +) +def test_restricted_expressions_reject_format_string_traversal(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +@pytest.mark.parametrize( + "source", + [ + # Generator/frame/code introspection reaches the eval globals and, via + # dict.update on them, could poison a shared namespace. None of these + # attribute names start with "_", so only the default-deny allowlist + # stops them. + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals))", + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_code))", + 'EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals.update({"P": 1})))', + # `mro` is a public method on the type metaclass; it reaches `object`. + "EvalResult(score=Score(0.5), reasoning=str(str.mro()[-1]))", + "EvalResult(score=Score(0.5), reasoning=str(int.mro()))", + # A live function's identity would leak a heap pointer (ASLR defeat). + "EvalResult(score=Score(0.5), reasoning=str(EvalResult.result_items))", + ], +) +def test_restricted_expressions_reject_introspection_attributes(source): + with pytest.raises(UnsafeEvaluatorSource, match="attribute"): + compile_evaluator(source) + + +def test_no_reachable_construct_leaks_a_heap_pointer_repr(): + # An object's default repr (`<... object at 0x...>`) leaks a live host heap + # address (ASLR/memory-layout disclosure) if coerced into a result field. + # Generator expressions are rejected at compile; `enumerate` is not bound, so + # it raises NameError at run time and becomes a bounded failed run instead of + # a disclosure. Either way, a pointer must never reach a result string. + import re + + from failproofai_sdk.evaluator.source import _SAFE_GLOBALS + + with pytest.raises(UnsafeEvaluatorSource, match="GeneratorExp"): + compile_evaluator("EvalResult(score=Score(1.0), reasoning=str((x for x in [1])))") + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(enumerate([1])))" + ) + with pytest.raises(NameError): + evaluate(Session()) + + # No value bound into the evaluation namespace reprs to a heap pointer. + pointer = re.compile(r"0x[0-9a-fA-F]+") + leaky = { + name: repr(value) + for name, value in _SAFE_GLOBALS.items() + if name != "__builtins__" and pointer.search(repr(value)) + } + assert leaky == {} + + +@pytest.mark.parametrize( + "inner", + [ + # A bound method's repr is `<... at 0xADDR>` — a live heap pointer. These + # methods stay allowlisted (real evals CALL them), but referencing one as a + # bare VALUE only serves to stringify that repr, so it is now rejected at + # COMPILE — the disclosure is closed at its source, not at the output. + "session.events[0].payload.get", + "''.join", + "'x'.encode", + "'a,b'.split", + ], +) +def test_bare_bound_method_reference_is_rejected_at_compile(inner): + for field in ( + f'EvalResult(score=Score(1.0), reasoning=str({inner}))', + f'EvalResult(score=Score(1.0), summary=str({inner}))', + f'EvalResult(score=Score(1.0, display_value=str({inner})))', + f'EvalResult(score=Score(1.0), labels=(str({inner}),))', + ): + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(field) + + +def test_heap_pointer_output_guard_bypasses_are_closed_at_compile(): + # An adversarial-review finding: the output-boundary regex was anchored on `<`, + # so a managed source could keep the address while reshaping the wrapper text — + # str(...).replace("<",""), an f-string, or %-formatting all coerce a bound + # method at a point the old scan missed. Each needs a BARE bound-method + # reference, which the compile-time call-site rule now rejects outright. + bypasses = [ + # str(...).replace("<","") strips the old regex's `<` anchor. + 'EvalResult(score=Score(1.0), reasoning=str(dict().get).replace("<", ""))', + # f-strings coerce at the C level, past the `str` global. + 'EvalResult(score=Score(1.0), reasoning=f"{session.events[0].payload.get}")', + # %-formatting coerces at the C level too. + 'EvalResult(score=Score(1.0), reasoning="%s" % session.events[0].payload.get)', + ] + for src in bypasses: + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(src) + + +def test_called_methods_and_data_attributes_still_stringify(): + # The call-site rule blocks only BARE method references. Calling methods and + # reading data attributes (both pointer-free) must still work — including the + # f-string and %-formatting paths — so legitimate evaluations are unaffected. + reasoning_call = compile_evaluator( + 'EvalResult(score=Score(1.0), ' + 'reasoning=str(session.events[0].payload.get("tool_name")))' + )(Session()) + assert reasoning_call.reasoning == "search" + + fstring = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning=f"n={session.event_count}")' + )(Session()) + assert fstring.reasoning == "n=3" + + percent = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning="pct=%d" % (session.event_count * 10))' + )(Session()) + assert percent.reasoning == "pct=30" + + +def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next(): + # Even setting aside the allowlist, one evaluation must not be able to leave + # state behind for the next. Compiling and running twice must not share a + # mutable namespace. + from failproofai_sdk.evaluator.source import _fresh_globals + + first = _fresh_globals() + second = _fresh_globals() + assert first is not second + assert first["__builtins__"] is not second["__builtins__"] + first["__poison__"] = "leaked" + assert "__poison__" not in second + + +def test_format_denylist_does_not_block_legitimate_string_methods(): + # The fix is a targeted denylist of `format`/`format_map`, not a ban on all + # string methods — ordinary evaluations must still compile and run. + evaluate = compile_evaluator( + 'EvalResult(score=Score(0.9, passed=True), ' + 'reasoning="tools=" + str(session.event_count).upper())' + ) + result = evaluate(Session()) + assert result.reasoning == "tools=3" + + +def test_restricted_expressions_reject_statements_and_oversized_source(): + with pytest.raises(UnsafeEvaluatorSource, match="one expression"): + compile_evaluator("import os") + with pytest.raises(UnsafeEvaluatorSource, match="exceeds"): + compile_evaluator("x" * (MAX_EVALUATOR_SOURCE_BYTES + 1)) + + +def test_result_and_condition_types_are_checked_at_runtime(): + with pytest.raises(TypeError, match="EvalResult"): + compile_evaluator("True")(Session()) + with pytest.raises(TypeError, match="bool or ConditionResult"): + compile_condition("1")(Session()) + + +def test_source_checksum_covers_condition_and_evaluator_together(): + base = source_checksum(None, "EvalResult()") + assert base == source_checksum(None, "EvalResult()") + assert base != source_checksum("True", "EvalResult()") + assert base != source_checksum(None, "EvalResult(summary='changed')") + + +# --- SEC-001: managed source cannot exhaust the worker (killable-fork sandbox) --- + + +def test_compute_bomb_is_killed_within_its_budget(): + # `sum(range(10**9))` would burn CPU for ~20s in-process, uncancellable — a + # CPU-bound loop (not a big allocation) so the wall-clock/CPU budget is what + # stops it, deterministically, rather than the memory ceiling. The forked + # sandbox kills it at its budget. + import time as _time + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + started = _time.monotonic() + with pytest.raises(EvaluationTimeout): + evaluate(Session()) + assert _time.monotonic() - started < 5 # bounded by the ~1s budget, not ~20s + + +def test_condition_compute_bomb_is_also_bounded(): + # `10**9`, matching the evaluator bomb above, NOT `10**8`. + # + # The property under test is "a CPU bomb in a condition is stopped by the + # sandbox budget", and the bomb has to be big enough that it cannot finish + # inside that budget on ANY machine the suite runs on. At 10**8 it was only + # ~1.35 CPU-seconds against a 1-second budget — a 1.35x margin — so on a fast + # runner the sum simply completed and nothing timed out. It failed exactly + # that way on CI under Python 3.14, which is faster here than 3.13 (1.35s vs + # 1.44s measured), while passing locally: a machine-speed coin flip, not a + # real signal about the sandbox. + # + # 10**9 restores the ~13x margin the evaluator twin already had. It costs no + # extra wall-clock: the sandbox kills the child at its budget either way, so + # a bigger bomb only widens the gap between "killed" and "could have + # finished". Do not shrink it back. + condition = compile_condition("sum(range(10**9)) > 0", timeout_seconds=1) + with pytest.raises(EvaluationTimeout): + condition(Session()) + + +def test_literal_pow_exponent_bomb_is_rejected_at_compile(): + with pytest.raises(UnsafeEvaluatorSource, match="exponent"): + compile_evaluator(f"EvalResult(score=Score(10 ** {MAX_POW_EXPONENT + 1}))") + # A small constant exponent stays allowed. + result = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(2 ** 3))" + )(Session()) + assert result.reasoning == "8" + + +def test_oversized_expression_is_rejected_at_compile(): + huge = "[" + ",".join("1" for _ in range(MAX_AST_NODES)) + "]" + with pytest.raises(UnsafeEvaluatorSource, match="too large"): + compile_evaluator(f"EvalResult(score=Score(len({huge}) / len({huge})))") + + +def test_normal_managed_eval_survives_the_fork_boundary(): + # A session-dependent result must round-trip out of the forked child intact. + result = compile_evaluator( + "EvalResult(score=Score(1.0 if session.event_count > 0 else 0.0), " + "reasoning=str(session.event_count))" + )(Session()) + assert isinstance(result, EvalResult) + assert result.score.value == 1.0 + assert result.reasoning == "3" + + +def test_sandbox_fails_closed_without_a_serializable_transcript(): + # The transcript crosses into the subprocess via `to_wire`. A session that + # can't be serialized cannot be sandboxed, so refuse rather than run unbounded. + class NotATranscript: + event_count = 1 + + with pytest.raises(EvaluationSandboxUnavailable): + compile_evaluator("EvalResult(score=Score(1.0))")(NotATranscript()) + + +def test_server_timeout_cannot_exceed_the_hard_ceiling(): + # SEC-001: a large server-provided timeout must not remove the execution bound. + assert _clamp_budget(10**9) == float(MAX_SANDBOX_TIMEOUT_SECONDS) + assert _clamp_budget(0) == 30.0 + assert _clamp_budget(None) == 30.0 + assert _clamp_budget(5) == 5.0 + + +def test_oversized_result_is_rejected_before_it_crosses_back(): + # A result with far more than the 25-item limit must be rejected INSIDE the + # sandbox (via result_items), so a huge result can never be serialized and + # shipped back to OOM the worker (SEC-001). + src = ( + "EvalResult(score=Score(1.0), " + "metrics={'m' + str(i): float(i) for i in range(200)})" + ) + with pytest.raises(ValueError, match="at most"): + compile_evaluator(src, eval_key="q")(Session()) + + +def test_sandbox_slot_wait_counts_against_the_timeout(monkeypatch): + # SEC-001: acquiring a concurrency slot must count against the wall-clock budget. + # `asyncio.wait_for` only cancels the awaiter, so a run that blocked UNBOUNDED on + # a busy slot would still launch a sandbox after its caller was reported timed + # out — 28 threads could queue behind 4 long sandboxes and starve the worker. + # With one slot and three 1s compute bombs, all three must resolve within ~one + # budget (the holder is killed at ~1s; the two queued behind it exhaust their + # budget waiting and time out WITHOUT ever spawning a child), not three serialized + # budgets (~3s). + import threading + import time as _time + + from failproofai_sdk.evaluator import source as _source + + monkeypatch.setattr(_source, "_SANDBOX_SLOTS", threading.Semaphore(1)) + bomb = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + session = Session() + errors: list[str] = [] + + def run(): + try: + bomb(session) + except Exception as error: # noqa: BLE001 + errors.append(type(error).__name__) + + threads = [threading.Thread(target=run) for _ in range(3)] + started = _time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + elapsed = _time.monotonic() - started + + assert errors == ["EvaluationTimeout"] * 3, errors + # Bounded by ~one budget, NOT three serialized ones — proving the queued runs + # timed out on slot acquisition instead of each waiting then running in turn. + assert elapsed < 2.5, f"queued sandboxes were not bounded by the timeout: {elapsed:.2f}s" + + +def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit(): + # A ~1.6 GiB allocation exceeds the per-sandbox RLIMIT_AS and is killed, so it + # cannot exhaust the worker even wrapped in an otherwise-valid result. With the + # concurrent-sandbox cap this also bounds the aggregate across concurrent runs. + src = "EvalResult(score=Score(1.0 if len([0] * 200000000) >= 0 else 0.0))" + with pytest.raises((EvaluationTimeout, MemoryError)): + compile_evaluator(src, timeout_seconds=5)(Session()) + + +def test_sandbox_fails_closed_when_kernel_resource_limits_are_unavailable(monkeypatch): + # SEC-001: on a platform without the stdlib ``resource`` module (e.g. Windows), + # the sandbox child cannot install RLIMIT_CPU / RLIMIT_AS on itself, so managed + # source must be refused BEFORE any child is spawned rather than run unbounded. + import failproofai_sdk.evaluator.source as source + + monkeypatch.setattr(source, "_resource", None) + + def _no_spawn(*args, **kwargs): + raise AssertionError("a sandbox child must not be started when limits are unavailable") + + monkeypatch.setattr(source.subprocess, "Popen", _no_spawn) + + with pytest.raises(EvaluationSandboxUnavailable): + source.compile_evaluator("EvalResult(score=Score(1.0))")(Session()) + + +def test_comprehension_body_can_read_session(): + # A list/set/dict comprehension resolves a free name like `session` from + # GLOBALS. When `session` was only in eval locals, such a source raised + # NameError on CPython 3.10 (a supported version). Both sandbox paths must + # now evaluate a session-dependent comprehension (COR-001). + assert ( + compile_condition("len([session.event_count for i in range(1)]) > 0")(Session()) + is True + ) + result = compile_evaluator( + "EvalResult(score=Score(1.0), " + "reasoning=str([session.event_count for i in range(1)]))" + )(Session(event_count=3)) + assert result.reasoning == "[3]" diff --git a/sdk/python/tests/test_zero_dependencies.py b/sdk/python/tests/test_zero_dependencies.py index a27fd75f..95be7b72 100644 --- a/sdk/python/tests/test_zero_dependencies.py +++ b/sdk/python/tests/test_zero_dependencies.py @@ -300,6 +300,23 @@ def test_importing_the_package_loads_no_framework(): ) +def test_importing_the_package_does_not_load_the_evaluator_runtime(): + """Telemetry-only users do not pay for the separate worker surface.""" + import json + import subprocess + + probe = ( + "import json, sys; import failproofai_sdk; " + "print(json.dumps(sorted(m for m in sys.modules " + "if m.startswith('failproofai_sdk.evaluator'))))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, cwd=str(ROOT), timeout=60 + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip()) == [] + + def test_the_adapter_registry_holds_strings_not_modules(): """`_REGISTRY` maps a name to a dotted path; importing it here would defeat it.""" from failproofai_sdk.integrations import _REGISTRY