diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 71e9f6c21..0688bc587 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -55,4 +55,135 @@ jobs: - name: Run unit tests with pytest run: | source .venv/bin/activate - pytest -n 16 + # `codex_smoke` is excluded here rather than relying on its + # CODEX_RUN_SMOKE opt-in alone: it spawns a real Codex subprocess and + # binds two real loopback ports, which must never run under `-n 16`. + pytest -n 16 -m "not codex_smoke" + + # Real Codex binary + real OS sandbox + real shim socket, against a stubbed + # model backend (no credentials, no network egress). Kept out of the matrix + # job above because it is serial and process-spawning. + # + # The job tolerates the test FAILING but not the test being ABSENT, and those + # are two separate steps below: + # + # * `continue-on-error` sits on the pytest step only. Whether the OS sandbox + # (landlock+seccomp on a GitHub runner) establishes at all is precisely the + # unknown this test exists to discover, and the CLI is pinned to an alpha + # (openai-codex-cli-bin==0.137.0a4), so a red assertion is signal to read, + # not a broken build. Drop that flag once the Linux sandbox verdict is in + # and the test has been green for a few runs -- at which point this job + # becomes an ordinary required check. + # * The step after it has NO such flag: a skipped or uncollected smoke test + # means the job proved nothing (lost binary, SDK install failure, platform + # check tripping), and an exit-0 green tick for that would defeat the whole + # point of running it. That must stay a hard failure even before the flag + # above comes off. + codex-smoke: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + + - name: Install dependencies + run: | + uv venv .venv + source .venv/bin/activate + uv sync --all-extras + uv pip install -e . + + - name: Run the Codex end-to-end smoke test + # Tolerates a failing assertion (the sandbox verdict this job exists to + # collect); does NOT tolerate the test not running -- see the next step. + continue-on-error: true + env: + CODEX_RUN_SMOKE: "1" + run: | + source .venv/bin/activate + # -p no:xdist: this test must not be distributed across workers. + # -rs: print skip reasons, so a silently skipped smoke test (missing + # SDK, missing binary, unsupported platform) is visible in the log. + # --junitxml: machine-readable outcome for the assertion step below; + # pytest's own exit code cannot distinguish "passed" from "skipped". + pytest -m codex_smoke -p no:xdist -rs --junitxml=codex-smoke.xml + + - name: Assert the smoke test actually ran + # No continue-on-error: this is the guard that stops the job going green + # while proving nothing. + run: | + python3 - <<'PY' + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + report = Path("codex-smoke.xml") + if not report.exists(): + sys.exit( + "codex smoke: pytest wrote no JUnit report, so it never got as " + "far as running tests (crash, bad invocation, or a broken venv)." + ) + + entries = list(ET.parse(report).getroot().iter("testcase")) + + def _detail(case, tag): + node = case.find(tag) + if node is None: + return None + name = f"{case.get('classname', '')}::{case.get('name', '')}" + return f" {name}: {node.get('message') or (node.text or '').strip()}" + + # pytest also emits a classname-less entry per module skipped during + # collection -- including modules `-m codex_smoke` deselected entirely + # (any `pytest.importorskip` at module scope elsewhere in the tree). + # Those are not this job's business; only real test cases are. + cases = [entry for entry in entries if entry.get("classname")] + if not cases: + collection = [d for d in (_detail(e, "skipped") for e in entries) if d] + sys.exit( + "codex smoke: no test ran under `-m codex_smoke`. Either the " + "smoke test was renamed/moved/lost its marker, or its whole " + "module was skipped at collection -- either way this job is " + "asserting nothing." + + ("\nmodules skipped at collection:\n" + "\n".join(collection) + if collection else "") + ) + + skipped = [d for d in (_detail(c, "skipped") for c in cases) if d] + if skipped: + sys.exit( + "codex smoke: the smoke test SKIPPED, so nothing was verified. " + "CODEX_RUN_SMOKE=1 is set by this job, so the cause is the " + "environment (openai-codex SDK missing, no runnable Codex " + "binary, or the platform check tripping) and must be fixed " + "rather than tolerated:\n" + "\n".join(skipped) + ) + + errored = [d for d in (_detail(c, "error") for c in cases) if d] + if errored: + sys.exit( + "codex smoke: the test errored in setup/teardown rather than " + "running to a verdict:\n" + "\n".join(errored) + ) + + failed = [d for d in (_detail(c, "failure") for c in cases) if d] + if failed: + # Deliberately not fatal: the pytest step above already reported it, + # and its `continue-on-error` is what keeps this job advisory while + # the Linux sandbox verdict is unknown. Both come off together. + print( + f"codex smoke: {len(cases)} test(s) ran, {len(failed)} FAILED " + "-- this is the signal this job exists to collect; read the " + "pytest output above:\n" + "\n".join(failed) + ) + else: + print(f"codex smoke: {len(cases)} test(s) ran and passed.") + PY diff --git a/docs/content/docs/framework/agent/runtime.en.mdx b/docs/content/docs/framework/agent/runtime.en.mdx index bd73914fc..ae75db426 100644 --- a/docs/content/docs/framework/agent/runtime.en.mdx +++ b/docs/content/docs/framework/agent/runtime.en.mdx @@ -28,7 +28,13 @@ agent = Agent(name="assistant", runtime="codex") | Value | Description | | :--- | :--- | | `adk` (default) | Google ADK's built-in execution flow; fits almost every case. | -| `codex` | Uses the OpenAI Codex SDK to drive the inner loop. | +| `codex` | Uses the OpenAI Codex SDK to drive the inner loop, executing commands and file edits inside a sandbox. | +| `piagent` | Uses a local Pi coding agent binary (through its RPC mode) to drive the inner loop. The binary is downloaded on first run, or point `PIAGENT_BINARY` at an existing executable. | + + +**Positioning: `codex` and `piagent` are *sandboxed-execution* runtimes, not drop-in equivalents of the ADK flow.** +They hand the entire inner loop to an external agent harness, so most things ADK implements *inside* that flow — request/response processors (planner, code executor, example store, knowledge base injection), per-LLM-call callbacks and spans — do not run. Basic agent transfer is bridged back into ADK through `transfer_to_agent`, but these runtimes are still best when you want a model that can run commands and edit files under a controlled sandbox, not when you need the full ADK flow. See the [support matrix](#support-matrix) for the exact differences. + Install the optional dependency before using the Codex runtime: @@ -36,9 +42,49 @@ Install the optional dependency before using the Codex runtime: pip install "veadk-python[codex]" ``` +### When to use the Codex runtime + +The positioning callout above reduces to a single test: **the Codex runtime's +one real advantage is that the model can write a file, run it, read the +traceback, fix it, and run it again — inside an OS sandbox, without you having +to pre-enumerate every step as a tool.** On every other axis it costs more than +`runtime="adk"`. + +**Good fits** + +- **Work whose steps cannot be enumerated as tools ahead of time** — ad-hoc analysis, log triage, data wrangling, code modification. What to do next depends on the last step's output, not on what you imagined while writing the agent. +- **Work over a filesystem** — intermediate artifacts stay in the workspace and are still there on the next turn of the same session (see [Workspace lifetime](#workspace-lifetime)). +- **Work where a wrong first attempt is normal and retrying is cheap** — the error message *is* the model's next input. Under the ADK runtime a failed tool call is just an error string; here it is a traceback the model can read, diagnose and fix. + +**Bad fits** + +- **A fixed tool call and a formatted answer.** That is the ADK runtime's home ground: faster, cheaper, and subject to none of the limits below. +- **An agent that needs `output_schema`, `planner`, `code_executor` or advanced `generate_content_config`.** Under a non-adk runtime these **raise** — they do not degrade. See the [support matrix](#support-matrix) for the full list. +- **Latency-sensitive paths.** Every invocation builds a fresh `CODEX_HOME`, spawns a Codex subprocess and opens an ephemeral thread; that fixed cost is paid per turn. +- **`run_live`.** Non-adk runtimes have no live/bidi implementation — see [below](#run_live-is-unsupported). + +**The costs, stated honestly** + +- **One Codex subprocess per invocation.** The runtime rebuilds `CODEX_HOME` and enters `AsyncCodex(...)` every turn, with an `ephemeral=True` thread. +- **The whole ADK history is re-serialized into the prompt every turn.** Prior turns are rendered as a `` JSON block in front of the current message — there is no incremental continuation — so prompt tokens grow with conversation length. It is also why `include_contents="none"` raises instead of being ignored. +- **ADK tool results travel through the model's context.** See [The workspace is the data plane](#the-workspace-is-the-data-plane) — the easiest way to ruin a turn with this runtime. + +The one-line version: **if you can express the task as a fixed sequence of tool +calls, use `runtime="adk"`; reach for `runtime="codex"` only when you can +describe the goal but the path has to be discovered.** + +The repository's four codex examples divide up like this: + +| Example | What it teaches | +| :--- | :--- | +| [`codex_data_analysis`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_data_analysis) | **What the runtime is for.** Codex writes an analysis script, runs it, hits a real error in dirty data, fixes it, re-runs, and reports — the self-iteration loop. | +| [`codex_ops_assistant`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_ops_assistant) | **What the runtime is for.** ADK tools land logs, metrics and deploy history in the workspace; Codex writes throwaway scripts to correlate the three and find a root cause, all inside a no-network sandbox. | +| [`codex_with_skill_and_mcp`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_with_skill_and_mcp) | **How to wire it up.** The paths a local skill and an MCP tool take under this runtime. | +| [`codex_runtime_on_agentkit`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_runtime_on_agentkit) | **How to deploy it.** Ship a `runtime="codex"` agent to Volcengine AgentKit. | + ### Codex security configuration -Codex defaults to a session-isolated workspace, the `workspace_write` sandbox, no network access, and denial of escalated operations. Broader access must be enabled explicitly: +Codex defaults to a session-isolated workspace, the `workspace_write` sandbox, no network access, and denial of every escalated operation: ```python title="agent.py" from veadk import Agent @@ -49,7 +95,7 @@ agent = Agent( runtime="codex", codex_runtime_config=CodexRuntimeConfig( sandbox="workspace_write", - approval_mode="auto_review", + approval_mode="deny_all", # the default; auto_review means full auto-approval, see below network_access=True, # Set this explicitly to work inside an existing project. workspace_root="/workspace/codex", @@ -57,36 +103,239 @@ agent = Agent( ) ``` -`full_access` and `reuse_workspace=True` relax filesystem isolation and should only be enabled in trusted environments. + +**`approval_mode="auto_review"` is not a review gate — it is full auto-approval.** +The Codex SDK's built-in approval handler answers every `requestApproval` notification with `accept`, and `AsyncCodex` exposes no hook to replace it. So `auto_review` auto-approves every sandbox escalation and file change without consulting a human or ADK. Only the default `deny_all` actually keeps Codex inside the sandbox. Do not use `auto_review` for multi-tenant deployments or untrusted input. + -### Codex observability +One sandbox/network combination is easy to misread: `network_access` is written to the `[sandbox_workspace_write]` table of Codex's `config.toml`, and **only the `workspace_write` sandbox reads it**. -Codex-native lifecycle notifications and ADK Function/MCP tool calls are converted into ADK Events. Runtime logs use stable `codex_*` event names and attribution fields such as `invocation_id`, `call_id`, `tool`, `status`, and `duration_ms`. Tool arguments, tool results, API tokens, credentials, and backend addresses are not logged. Token usage is exposed through `codex_event_type=token_usage` events and the corresponding log entry. +| `sandbox` | Is `network_access` honoured? | +| :--- | :--- | +| `workspace_write` | Yes. | +| `read_only` | No (setting it to `True` logs a warning). | +| `full_access` | No — `danger-full-access` already means full network and filesystem access. | -## Choose where each tool runs +Because of that, `CodexRuntimeConfig(sandbox="full_access", network_access=False)` **raises `ValueError`**: the combination reads as "no network" while actually granting full access. If you need `full_access`, write `network_access=True` explicitly to acknowledge the risk. -`RuntimeProvider` leaves the Agent's model reasoning loop unchanged and only decides where individual tool calls execute. `DispatchRuntimeProvider` can send selected tools to a remote runtime while falling back to their original ADK implementation for all other tools: +`full_access` and `reuse_workspace=True` relax filesystem isolation between invocations and should only be enabled in trusted environments. + +#### The security baseline + +Put the pieces above together and you get the default recipe for running codex +over untrusted input. Each of the four settings blocks a different direction, +and none of them is redundant: ```python title="agent.py" +from google.adk.agents import RunConfig from veadk import Agent, Runner -from veadk.runtime import DispatchRuntimeProvider - -async def dispatch_task(tool_call): - return await remote_client.dispatch( - tool_call.name, - tool_call.arguments, - dispatch_id=tool_call.id, - ) - -agent = Agent(name="assistant", tools=[bash, read_file]) -runtime_provider = DispatchRuntimeProvider( - dispatch_task, - dispatchable_tools=None, # dispatch every non-MCP tool +from veadk.runtime.codex import CodexRuntimeConfig + +agent = Agent( + name="analyst", + runtime="codex", + tools=[load_orders, publish_report], # the only way in and out + codex_runtime_config=CodexRuntimeConfig( + sandbox="workspace_write", # writes confined to the workspace + network_access=False, # no outbound path from the sandbox + approval_mode="deny_all", # the default; refuses every escalation + ), +) + +await Runner(agent=agent).run( + "...", + run_config=RunConfig(max_llm_calls=40), # cost ceiling for the invocation ) -runner = Runner(agent=agent, plugins=[runtime_provider]) ``` -In this example, every non-MCP tool only goes through `dispatch_task` and its local function is not invoked a second time. MCP tools retain their original ADK implementation. Pass a set of names to dispatch only those non-MCP tools. The dispatcher may be synchronous or asynchronous. +They compose because **with the network off, the ADK tools you wired up are the +only outbound path**: the model can compute freely over sensitive data and retry +as often as it likes, but to get anything *out* it has to go through a function +you audited, signed and logged. Your tool list is your data-egress policy — which +holds only as long as no individual tool offers arbitrary outbound calls of its own. + +Three related points, covered elsewhere rather than repeated here: + +- `approval_mode="auto_review"` removes the foundation this recipe stands on; see the [error callout above](#codex-security-configuration). +- The `sandbox` × `network_access` semantics are in the [table above](#codex-security-configuration): only `workspace_write` reads `network_access`. +- The `VEADK_CODEX_*` environment variables take **precedence over** the Python config; see [Codex environment variables](#codex-environment-variables). A single `VEADK_CODEX_SANDBOX=full_access` in the deployment environment overrides all of the code above. + +When `RunConfig(max_llm_calls=...)` is unset, VeADK's `Runner` falls back to the +`MODEL_AGENT_MAX_LLM_CALLS` environment variable (default `100`). Under this +runtime, set it explicitly and lower: the model decides for itself how many +script rounds to run, and this budget is the only hard stop. + +#### Workspace lifetime + +By default a workspace is keyed by `app_name` / `user_id` / `session_id` / agent name: every invocation of one session **shares the same directory**, and it is not deleted at the end of a turn, so files written by one turn are still there for the next. The whole tree is removed when the process exits; while the process runs, session workspaces left **idle for more than 6 hours** are reaped the next time a workspace is created, which bounds disk growth in a long-lived server. + +A tool never has to derive that path: `current_workspace()` returns the workspace of the turn that is calling it, with nothing configured — see [The workspace is the data plane](#the-workspace-is-the-data-plane) below. + +A `workspace_root` you pass explicitly is *your* directory: the runtime never deletes it and never applies that idle reaping to it — it still creates a per-session subdirectory underneath, but cleaning those up is on you. `reuse_workspace=True` only means anything alongside `workspace_root`: it makes every session share that directory itself, with no per-session isolation. + +### The workspace is the data plane + +This is the most important technique for this runtime, and the easiest one to +miss: **tool arguments and tool results are the control plane; the workspace is +the data plane.** + +The reason is the path a tool call takes. ADK/MCP tools are not called by Codex +itself — the runtime's Responses shim executes them, serializes the return value +with `json.dumps`, and appends the resulting string to the model's `input` array +as a `function_call_output`. And because Codex rebuilds the entire `input` after each +of its own native tool rounds, the shim must **replay that +`function_call`/`function_call_output` pair into every subsequent backend +request of the same turn**. So: + + +A tool that returns 50k log lines pushes those 50k lines into the model's context — and pushes them again on each following request of that turn. This is not "a bit slower"; it is a wasted turn. + + +The fix is for a tool to hand back **a path, never a payload**: write the data +into the workspace (which is Codex's `cwd`) and return only its location plus a +little metadata. The data sits on disk where the model can grep, slice and +aggregate it with shell commands, and only one line of it reaches the context. + +```python title="tools.py" +from pathlib import Path + +from veadk.runtime.codex import current_workspace + + +def load_orders(day: str) -> dict: + """Write one day of orders into your working directory as CSV. + + Returns a receipt, never the rows themselves — read the CSV at the + returned path with your own code. + """ + workspace = current_workspace() # this turn's directory, or None + if workspace is None: # not a Codex turn: an error the model can read + return {"status": "error", "message": "no codex workspace on this call"} + rows = warehouse.query(day) + name = f"orders-{day}.csv" + write_csv(Path(workspace) / name, rows) + # A receipt, not a payload: where it is and what shape it has. + return {"path": name, "rows": len(rows), "columns": list(rows[0])} +``` + +The docstring is doing real work here: it is what the model reads, so it has to +say that the result is a receipt. Otherwise the model asks the tool for the data +and you are back where you started. + +The same holds in the other direction. Once Codex has written a report into the +workspace, a "publish" tool should take **the path of the file it wrote** and +read it off disk itself — not have the model re-dictate the whole report as a +tool argument, which pushes it through the context a second time and invites it +to drift on the way. + +How a tool knows where the workspace is: + +- **`current_workspace()` — the primary way, and the one multi-tenancy needs.** `from veadk.runtime.codex import current_workspace` gives the absolute path of the workspace for the turn that is calling the tool. It needs no configuration and works with `workspace_root` and `reuse_workspace` both **unset**. The value is bound around each tool call rather than read from ambient invocation state, so turns running concurrently in one process each see their own directory. It returns `None` — never raises — when no Codex turn is on the call stack, because the same tool object is also run by other runtimes, by `AgentTool`, and by unit tests; branch on that and return your own `{"status": "error", ...}`, which the model can act on, instead of raising or falling back to a directory of your own choosing. +- **A fixed path — single tenant, and you want to read the directory afterwards.** Set `workspace_root` together with `reuse_workspace=True`; the workspace is then `workspace_root` itself, so a tool and the `CodexRuntimeConfig` can share one constant and the directory is still on disk once the process has exited. The price is that every session shares it — see [Workspace lifetime](#workspace-lifetime). + +Either way, a path that came from the *model* is untrusted input: resolve it against the workspace and reject anything that escapes (`..`, absolute paths, symlinks pointing out) before opening it. + +### Codex execution knobs + +| Field | Default | Description | +| :--- | :--- | :--- | +| `reasoning_effort` | `"medium"` | Reasoning budget: `minimal`/`low`/`medium`/`high`/`xhigh`. Higher is slower and uses more tokens. | +| `personality` | `"pragmatic"` | Codex's own reply style: `none`/`friendly`/`pragmatic`. | +| `max_tool_iterations` | `32` | Budget of bridged ADK/MCP tool round-trips the shim may run for the whole Codex turn (1–256). See the behavior change below. | +| `tool_timeout_seconds` | `120.0` | Per-call timeout for a bridged ADK/MCP tool. `None` disables the timeout. | +| `reuse_workspace` | `False` | Only meaningful alongside `workspace_root`, and only worth setting for a single tenant — see [Workspace lifetime](#workspace-lifetime) above. Leave both unset and let tools call `current_workspace()`; see [The workspace is the data plane](#the-workspace-is-the-data-plane). | + + +**Behavior change: `max_tool_iterations` changed meaning, and its default went from `8` to `32`.** +It now bounds the **whole Codex turn**, not a single backend request. Codex issues a fresh backend request after every native tool round, so the old per-request counter actually allowed *rounds × budget* tool executions. The default was raised at the same time so that turns which reach for an ADK tool only after several native tool rounds are not cut short. If you were relying on `8` as a cost backstop, re-evaluate — `RunConfig(max_llm_calls=...)` (below) is now the better ceiling. + + +### How instructions reach Codex + +Codex ships its own ~20KB system prompt, tuned for its own toolchain (`apply_patch`, `update_plan`, shell, AGENTS.md). VeADK deliberately does **not** override it via `base_instructions`: Codex *replaces* the built-in template outright when that field is set, which would delete all of that guidance (Codex's own docs call the equivalent config key strongly discouraged). + +The agent's identity block and its `instruction` therefore travel over Codex's native `developer_instructions` channel instead. Codex renders that as its own `developer` message alongside AGENTS.md, skills and environment context — purely additive, never a replacement. + +One consequence: **`personality` now actually takes effect.** It is rendered into Codex's built-in system prompt template, and that template — personality included — used to be replaced wholesale whenever `base_instructions` was set, leaving the field inert. + +Two corrections ride along on the same channel, because Codex's preserved system prompt describes a toolchain this bridge cannot fully deliver. The runtime appends a short **tool-availability note** to every turn's developer instructions: + +- `apply_patch` is not one of the tools on this run — the shim forwards only `function`-typed tools, and Codex's file-editing tool is not one — so files are created and edited with `exec_command` (for example a `cat > file <<'EOF'` heredoc). +- `request_user_input` *is* advertised, but nothing can answer it: an ADK invocation has no interactive channel, so calling it ends the turn with the work undone. The model is told to decide with what it has and to say what was missing in its final message. + +You do not need to counter-instruct for either of these in your agent's `instruction`. + +### Codex environment variables + + +The four variables below **override** the Python `CodexRuntimeConfig` rather than acting as defaults for it. A hard-coded `sandbox="workspace_write"` in your code is still overridden by `VEADK_CODEX_SANDBOX=full_access` in the deployment environment. This precedence is the opposite of most settings and matters most where the platform injects environment variables (containers, FaaS). + + +| Variable | Overrides | Notes | +| :--- | :--- | :--- | +| `VEADK_CODEX_SANDBOX` | `sandbox` | Same values as the field. | +| `VEADK_CODEX_APPROVAL_MODE` | `approval_mode` | Same values as the field. | +| `VEADK_CODEX_WORKSPACE_ROOT` | `workspace_root` | Absolute path to the workspace. | +| `VEADK_CODEX_NETWORK_ACCESS` | `network_access` | `1`/`true`/`yes`/`on` enable it; anything else disables it. | + +Four more variables tune the Responses→Chat shim. These are not part of the override rule above — they only tune the shim itself: + +| Variable | Default | Notes | +| :--- | :--- | :--- | +| `CODEX_SHIM_NUM_RETRIES` | `2` | Retries for a backend request that fails transiently (429/5xx/overloaded/timeout). | +| `CODEX_SHIM_TIMEOUT` | `0` (no timeout) | Per-request timeout in seconds. | +| `CODEX_SHIM_START_TIMEOUT` | `10` | Seconds to wait for the shim's local HTTP server to come up. On timeout the startup fails loudly instead of leaving Codex unable to connect. | +| `CODEX_SHIM_CACHE_MAX` | `8` | Cap on cached shim instances per process (keyed by backend address + credential), bounding servers and ports in a multi-tenant process. | + +### Codex observability + +Codex-native lifecycle notifications and ADK Function/MCP tool calls are converted into ADK Events. Runtime logs use stable `codex_*` event names and attribution fields such as `invocation_id`, `call_id`, `tool`, `status`, and `duration_ms`. Tool arguments, tool results, API tokens, credentials, and backend addresses are not logged. Token usage is exposed through `codex_event_type=token_usage` events and the corresponding log entry. + +On top of that: + +- **A `call_llm` span.** ADK opens `call_llm` from its own LLM flow, which this runtime replaces, so the runtime opens the equivalent span itself and writes the prompt, the response and the token usage onto it at the end of the turn. VeADK's whole telemetry chain — the in-memory exporter's session index, trace export, portal metrics, and trace-based evaluation — therefore sees a Codex invocation at all. Mind the granularity: **one span per turn**, not one per inner model call. +- **`usage_metadata`.** The turn's token usage is summed and attached to the single merged final event of that turn, rather than to each intermediate event, so downstream consumers that sum `usage_metadata` across events do not double-count. +- **`RunConfig(max_llm_calls=...)` is enforced.** The shim charges the budget before every real backend model call. On exhaustion Codex sees a `429 llm_calls_limit` (not a 500, which it would retry — replaying every tool side effect of the turn), and once the turn ends the runtime re-raises `LlmCallsLimitExceededError` to the caller instead of returning whatever partial answer Codex salvaged. + +## Support matrix + +`runtime="codex"` and `runtime="piagent"` replace the whole ADK LLM flow, so a substantial part of the `Agent` configuration surface has no effect. VeADK checks for this at construction time and again before every invocation: **configuration that produces a wrong result raises `ValueError`**, and **configuration that is merely ignored logs a warning once**. + +### Error — the agent refuses to run + +| Configuration | Why | +| :--- | :--- | +| `model=...` | The runtime resolves the model from `model_name` and ignores the `model` object entirely, dropping its `api_base`, headers and fallbacks. Set `model_name`. | +| `generate_content_config=...` | Only `system_instruction` is forwarded; `temperature`, `max_output_tokens`, `thinking_config` and friends are dropped. | +| `output_schema=...` | The schema reaches neither the backend nor the prompt, so the model is never asked for it; `state[output_key]` would hold an unvalidated reply or silently be missing. | +| `planner=...` / `code_executor=...` | Both run as ADK request/response processors, a layer external runtimes never execute. | +| `include_contents="none"` | External runtimes always send the full conversation history, so this would be silently ignored and prior turns leaked to the model. | +| `enable_supervisor=True` | Supervision is installed through the ADK LLM flow, which the runtime replaces. | + + +`sub_agents` are supported through an ADK-compatible `transfer_to_agent` bridge. The external runtime can request a handoff, and VeADK then runs the selected target agent in the surrounding ADK context. + + +### Warning — ignored, but the turn still answers + +| Configuration | Behavior | +| :--- | :--- | +| `model_name=[primary, fallback...]` | Only the first entry is used; the fallback chain does not apply. | +| `model_provider` (non-`openai`) | The runtime always talks to `model_api_base` over an OpenAI-compatible API. | +| `model_extra_config` | `piagent` only. **codex forwards it**: the shim puts `extra_headers` and `extra_body` — including VeADK's Ark defaults for request encryption and prompt caching — onto the backend request, matching the `adk` path. | +| `enable_responses` / `enable_responses_cache` | The Ark Responses API is unused, so `previous_response_id` continuation and response caching do not apply. | +| `example_store` | Delivered by `ExampleTool.process_llm_request`, a hook external runtimes never call, so no few-shot examples reach the model. | +| `knowledgebase` | **The knowledge base is silently disabled.** Same root cause: `LoadKnowledgebaseTool.process_llm_request` is what tells the model the knowledge base exists and when to query it, and that hook is never called, so retrieval is never triggered. | +| `skills_mode` | External runtimes skip VeADK's `SkillsToolset` when bridging tools, so the `execute_skills`/`skills_tool` the instruction advertises is not registered. | +| `enable_skills_checklist` | Depends on an ADK before-tool callback over the skills toolset; neither is bridged. | +| `after_model_callback` | Different semantics: it fires once per turn on the merged final text, not once per LLM call. | +| `tracers` | Different granularity: codex does add a `call_llm` span (prompt, response, whole-turn token usage), but **one per turn** — the inner model loop runs inside the external harness, so there are still no per-model-call spans. | +| `codex_runtime_config` (with `runtime="piagent"`) | Codex-only; piagent ignores it entirely. | +| `RunConfig(max_llm_calls=...)` | `piagent` only — its model loop counts its own iterations inside the harness and the budget does not bind. **codex enforces it**; see [Codex observability](#codex-observability) above. | + +### `run_live` is unsupported + +Non-`adk` runtimes have no live/bidi implementation. `/run_live`, exposed by ADK's `get_fast_api_app` and used by `veadk web`, raises `NotImplementedError` for such an agent instead of silently falling back to the ADK flow — which would run a different model loop with a different tool set. Use `runtime="adk"` for live sessions, and `runner.run_async` everywhere else. ## Request processing diff --git a/docs/content/docs/framework/agent/runtime.mdx b/docs/content/docs/framework/agent/runtime.mdx index a5630d9d4..bfac65e82 100644 --- a/docs/content/docs/framework/agent/runtime.mdx +++ b/docs/content/docs/framework/agent/runtime.mdx @@ -28,7 +28,13 @@ agent = Agent(name="assistant", runtime="codex") | 取值 | 说明 | | :--- | :--- | | `adk`（默认） | 使用 Google ADK 内置的执行流程，适用于绝大多数场景。 | -| `codex` | 使用 OpenAI Codex SDK 驱动内层循环。 | +| `codex` | 使用 OpenAI Codex SDK 驱动内层循环，在沙箱中执行命令与文件操作。 | +| `piagent` | 使用本地 Pi coding agent 二进制（通过其 RPC 模式）驱动内层循环。二进制会在首次运行时自动下载，也可用 `PIAGENT_BINARY` 指定已有路径。 | + + +**定位：`codex` / `piagent` 是「沙箱执行运行时」，不是 ADK 执行流程的等价替代品。** +它们把整个内层循环交给外部 agent harness，因此 ADK 在该流程内实现的大部分能力——请求/响应处理器（planner、code_executor、example store、知识库注入）、逐次模型调用的回调与 span——都不会执行。基础 Agent transfer 会通过 `transfer_to_agent` 桥接回 ADK，但这些运行时仍更适合「模型可以在受控沙箱里跑命令、改文件」的场景，而不是需要完整 ADK flow 的场景。具体差异见下方[支持矩阵](#支持矩阵)。 + 使用 Codex runtime 前安装可选依赖： @@ -36,9 +42,43 @@ agent = Agent(name="assistant", runtime="codex") pip install "veadk-python[codex]" ``` +### 什么时候该用 codex 运行时 + +上面的定位说明可以压成一条判断标准：**codex 运行时只有一个真正的优势——模型可以写下一个文件、把它跑起来、读到报错、改掉、再跑一遍，整个循环发生在一个 OS 沙箱里，而你不需要事先把每一步都定义成工具。** 除此之外的每一条轴上，它都比 `runtime="adk"` 更贵。 + +**适合：** + +- **步骤无法事先枚举成工具**——临时数据分析、日志排障、脏数据清洗、按需改代码。这类任务的下一步取决于上一步的输出，而不是取决于你写智能体时的设想。 +- **工作对象是一个文件系统**——中间产物留在 workspace 里，同一 Session 的下一轮仍然看得见（见 [Workspace 生命周期](#workspace-生命周期)）。 +- **第一次做错是正常的，且重试很便宜**——报错信息本身就是模型的下一份输入。在 ADK 运行时里，一次失败的工具调用只是一条错误字符串；在这里，它是一段可以被读、被诊断、被修好的 traceback。 + +**不适合：** + +- **一次固定的工具调用 + 一段格式化回答。** 这正是 ADK 运行时的主场：更快、更便宜，而且不受下面任何一条限制。 +- **需要 `output_schema`、`planner`、`code_executor` 或高级 `generate_content_config` 的智能体。** 这些配置在非 adk 运行时下会**直接报错**，而不是降级——完整清单见[支持矩阵](#支持矩阵)。 +- **对延迟敏感的链路。** 每一次 invocation 都会重建一个临时 `CODEX_HOME`、拉起一个 Codex 子进程、开一个 ephemeral thread；这笔固定开销每回合都要付。 +- **`run_live`。** 非 adk 运行时没有 live/bidi 实现，见[下文](#不支持-run_live)。 + +**成本，如实说清楚：** + +- **每次 invocation 一个 Codex 子进程。** runtime 每回合都会重建 `CODEX_HOME` 并进入 `AsyncCodex(...)`，thread 以 `ephemeral=True` 创建。 +- **每回合都会把整段 ADK 会话历史重新序列化进 prompt。** 历史被渲染成一个 `` JSON 块，附在当前消息前面；codex 没有增量续接机制，所以 prompt token 随对话轮数增长。这也是 `include_contents="none"` 被拒绝（而不是被忽略）的原因。 +- **ADK 工具的结果会穿过模型上下文。** 见下方 [Workspace 是数据面](#workspace-是数据面)——这是本运行时最容易踩的一个坑。 + +一句话版本：**能把任务写成一串确定的工具调用，就用 `runtime="adk"`；只有当你只能描述目标、路径必须试出来时，才用 `runtime="codex"`。** + +仓库里四个 codex 示例的分工： + +| 示例 | 讲什么 | +| :--- | :--- | +| [`codex_data_analysis`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_data_analysis) | **本运行时是干什么用的。** Codex 写分析脚本 → 运行 → 在脏数据上撞到真实报错 → 自己改好 → 重跑 → 出报告。自迭代循环。 | +| [`codex_ops_assistant`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_ops_assistant) | **本运行时是干什么用的。** ADK 工具把日志、指标与发布记录落进 workspace，Codex 写一次性脚本把三者关联起来、定位根因，全程在断网沙箱里。 | +| [`codex_with_skill_and_mcp`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_with_skill_and_mcp) | **怎么接线。** 本地 skill 和 MCP 工具在这个运行时下分别走哪条路。 | +| [`codex_runtime_on_agentkit`](https://github.com/volcengine/veadk-python/tree/main/examples/codex_runtime_on_agentkit) | **怎么部署。** 把一个 `runtime="codex"` 智能体发布到火山引擎 AgentKit。 | + ### Codex 安全配置 -Codex 默认使用独立 Session workspace、`workspace_write` 沙箱、禁用网络，并拒绝需要提权的操作。需要扩大权限时必须显式配置： +Codex 默认使用独立 Session workspace、`workspace_write` 沙箱、禁用网络，并拒绝一切需要提权的操作： ```python title="agent.py" from veadk import Agent @@ -49,7 +89,7 @@ agent = Agent( runtime="codex", codex_runtime_config=CodexRuntimeConfig( sandbox="workspace_write", - approval_mode="auto_review", + approval_mode="deny_all", # 默认值；auto_review 等同于全自动批准，见下 network_access=True, # 若需在已有工程内工作，显式指定目录；默认使用 Session 隔离目录。 workspace_root="/workspace/codex", @@ -57,36 +97,213 @@ agent = Agent( ) ``` -`full_access` 和 `reuse_workspace=True` 会放宽不同调用之间的文件系统边界，只应在受信环境中开启。 + +**`approval_mode="auto_review"` 不是「人工复核」，而是「全自动批准」。** +Codex SDK 内置的审批处理器对每一个 `requestApproval` 通知都回答 `accept`，而 `AsyncCodex` 没有提供替换该处理器的接口。因此 `auto_review` 会自动批准每一次沙箱提权和文件修改，既不询问人工，也不经过 ADK。只有默认的 `deny_all` 才真正把 Codex 约束在沙箱内。在多租户或不可信输入场景中，请勿使用 `auto_review`。 + -### Codex 可观测性 +关于沙箱与网络的组合，有一个容易误判的点：`network_access` 只会写入 Codex `config.toml` 的 `[sandbox_workspace_write]` 段，**只有 `workspace_write` 沙箱会读取它**。 -Codex 原生生命周期和 ADK Function/MCP 工具调用都会转换为 ADK Event。运行日志使用稳定的 `codex_*` 事件名，并包含 `invocation_id`、`call_id`、`tool`、`status`、`duration_ms` 等可归因字段。日志不会记录工具参数、工具结果、API Token、凭证或后端地址；Token Usage 通过 `codex_event_type=token_usage` 事件及对应日志提供。 +| `sandbox` | `network_access` 是否生效 | +| :--- | :--- | +| `workspace_write` | 生效。 | +| `read_only` | 不生效（设为 `True` 会记录一条警告）。 | +| `full_access` | 不生效——`danger-full-access` 本身就意味着完全的网络与文件系统访问。 | + +因此 `CodexRuntimeConfig(sandbox="full_access", network_access=False)` 会**直接抛出 `ValueError`**：这个组合读起来像「禁用网络」，实际却给了完全访问权限。需要 `full_access` 时必须显式写上 `network_access=True` 以确认风险。 -## 按工具选择执行位置 +`full_access` 和 `reuse_workspace=True` 会放宽不同调用之间的文件系统边界，只应在受信环境中开启。 -`RuntimeProvider` 不替换智能体的模型推理循环，只决定单次工具调用在哪里执行。`DispatchRuntimeProvider` 可以把指定工具派发到远端，其余工具回退到原本的 ADK 本地实现： +#### 安全基线组合 + +把上面几项拼起来，就是在不可信输入下跑 codex 的默认配方。四个设置各挡一个方向，缺一不可： ```python title="agent.py" +from google.adk.agents import RunConfig from veadk import Agent, Runner -from veadk.runtime import DispatchRuntimeProvider - -async def dispatch_task(tool_call): - return await remote_client.dispatch( - tool_call.name, - tool_call.arguments, - dispatch_id=tool_call.id, - ) - -agent = Agent(name="assistant", tools=[bash, read_file]) -runtime_provider = DispatchRuntimeProvider( - dispatch_task, - dispatchable_tools=None, # 派发所有非 MCP 工具 +from veadk.runtime.codex import CodexRuntimeConfig + +agent = Agent( + name="analyst", + runtime="codex", + tools=[load_orders, publish_report], # 唯一的进出通道 + codex_runtime_config=CodexRuntimeConfig( + sandbox="workspace_write", # 只能写 workspace + network_access=False, # 沙箱内没有出网通道 + approval_mode="deny_all", # 默认值；拒绝一切提权请求 + ), +) + +await Runner(agent=agent).run( + "...", + run_config=RunConfig(max_llm_calls=40), # 单次 invocation 的成本上限 ) -runner = Runner(agent=agent, plugins=[runtime_provider]) ``` -上例中所有非 MCP 工具只会经过 `dispatch_task`，不会再次执行本地函数；MCP 工具保留原本的 ADK 实现。传入具体工具名集合时，只派发集合中的非 MCP 工具。派发函数可以是同步或异步函数。 +它们之所以能组合成一套，是因为**关掉网络之后，你自己挂上去的那几个 ADK 工具就是唯一的出站通道**：模型可以在沙箱里对敏感数据任意计算、任意重试，但要把任何东西送出去，只能经过一个你审计过、有签名、有日志的函数。换句话说，工具列表就是你的数据出口策略——这条边界要成立，前提是每个工具自己也不提供任意外发能力。 + +配套的三条，不在这里重复展开： + +- `approval_mode="auto_review"` 会拆掉这套配方的地基，理由见[上方的红色警告](#codex-安全配置)。 +- `sandbox` 与 `network_access` 的组合语义见[上表](#codex-安全配置)：只有 `workspace_write` 会读 `network_access`。 +- `VEADK_CODEX_*` 环境变量的优先级**高于**这里的 Python 配置，见 [Codex 环境变量](#codex-环境变量)。部署环境里一个 `VEADK_CODEX_SANDBOX=full_access` 就能推翻上面整段代码。 + +`RunConfig(max_llm_calls=...)` 不设时，VeADK 的 `Runner` 会用环境变量 `MODEL_AGENT_MAX_LLM_CALLS`（默认 100）兜底。在这个运行时下建议显式写一个更小的值：模型自己决定跑多少轮脚本，只有这个配额是硬上限。 + +#### Workspace 生命周期 + +默认情况下，workspace 由 `app_name` / `user_id` / `session_id` / agent 名四元组定位：同一 Session 的多次调用**共享同一目录**，回合结束后不会删除，因此上一轮写下的文件下一轮仍然可见。进程退出时整棵目录树被清理；进程存活期间，**空闲超过 6 小时**的 Session workspace 会在下一次创建 workspace 时被顺带回收，以限制长期运行的服务进程的磁盘增长。 + +工具不需要自己推导这个路径：不做任何配置，`current_workspace()` 就会返回当前调用它的这一轮的 workspace，见下方 [Workspace 是数据面](#workspace-是数据面)。 + +显式传入 `workspace_root` 时，这块目录属于你自己：runtime 既不会删除它，也**不会**对它执行上述空闲回收——它仍会在其下为每个 Session 建独立子目录，但这些子目录需要你自行清理。`reuse_workspace=True` 只有在同时设置了 `workspace_root` 时才有意义，它让所有 Session 直接共用该目录本身，而不再做 Session 隔离。 + +### Workspace 是数据面 + +这是用好 codex 运行时最重要、也最容易被忽略的一条：**ADK 工具的参数和返回值是控制面，workspace 才是数据面。** + +原因在工具的执行路径上。ADK/MCP 工具不由 Codex 自己调用，而是由 runtime 的 Responses shim 执行：shim 拿到工具返回值后 `json.dumps` 成字符串，作为一条 `function_call_output` 塞回模型的 `input` 数组。而且因为 Codex 每完成一次原生工具调用就会重建整个 `input`，shim 还必须**在本回合后续的每一次后端请求里，把这对 `function_call` / `function_call_output` 重放一遍**。于是： + + +一个返回 5 万行日志的工具，会把这 5 万行灌进模型上下文，并在同一回合里重复灌若干次。这不是慢一点的问题，是这一回合直接废掉。 + + +正确的做法是让工具**交出一个路径，而不是一份数据**：把数据写进 workspace（Codex 的 `cwd` 就是它），只返回位置和少量元信息。数据落在磁盘上，模型用 shell 去读、去切片、去统计，上下文里只留下一行。 + +```python title="tools.py" +from pathlib import Path + +from veadk.runtime.codex import current_workspace + + +def load_orders(day: str) -> dict: + """把某一天的订单写成 CSV，放进你的工作目录。 + + 只返回一张回执，不返回订单本身——请用你自己的代码去读返回路径上的 CSV。 + """ + workspace = current_workspace() # 本轮的工作目录，或者 None + if workspace is None: # 不在 Codex 轮次里：返回模型能读懂的错误 + return {"status": "error", "message": "no codex workspace on this call"} + rows = warehouse.query(day) + name = f"orders-{day}.csv" + write_csv(Path(workspace) / name, rows) + # 回执，不是数据：告诉模型文件在哪、有多大形状。 + return {"path": name, "rows": len(rows), "columns": list(rows[0])} +``` + +这里的 docstring 是真正干活的部分：模型读到的就是它，所以必须写清楚「返回的是回执」。 +否则模型会直接让工具把数据交出来，你又回到了原点。 + +反方向同理：Codex 在 workspace 里写好报告之后，「发布」工具应该收下**它写的那个文件的路径**，由工具自己读盘再送往外部系统——而不是让模型把整篇报告当作工具参数复述一遍（那同样要过一遍上下文，而且内容可能在复述中走样）。 + +工具怎么知道 workspace 在哪： + +- **`current_workspace()`——首选做法，也是多租户唯一可行的做法。** `from veadk.runtime.codex import current_workspace`，返回当前调用该工具的这一轮 workspace 的绝对路径。它不需要任何配置，`workspace_root` 与 `reuse_workspace` 都**不设**时同样有效。这个值是在每次工具调用前后绑定的，而不是从调用方的环境上下文里读，所以同一进程里并发的多个回合各自看到自己的目录。调用栈上没有 Codex 回合时它返回 `None`，而不是抛异常——同一个工具对象也会被别的 runtime、被 `AgentTool`、被单元测试执行——因此请判断 `None` 并返回你自己的 `{"status": "error", ...}` 让模型去处理，而不是抛异常，也不要退回到某个你自己选定的目录。 +- **固定路径——单租户，并且你想在跑完之后翻这个目录。** 设置 `workspace_root` 并配合 `reuse_workspace=True`，此时 workspace 就是 `workspace_root` 本身，工具和 `CodexRuntimeConfig` 可以共用同一个常量，而且进程退出后目录仍在磁盘上。代价是所有 Session 共用它，见 [Workspace 生命周期](#workspace-生命周期)。 + +无论哪种做法，来自**模型**的路径都是不可信输入：打开之前请把它解析回 workspace，并拒绝一切越界的路径（`..`、绝对路径、指向外部的符号链接）。 + +### Codex 执行参数 + +| 参数 | 默认值 | 说明 | +| :--- | :--- | :--- | +| `reasoning_effort` | `"medium"` | 推理强度，可选 `minimal`/`low`/`medium`/`high`/`xhigh`。越高越慢、消耗 token 越多。 | +| `personality` | `"pragmatic"` | Codex 自带的回复风格，可选 `none`/`friendly`/`pragmatic`。 | +| `max_tool_iterations` | `32` | 整个 Codex turn 内 shim 可执行的 ADK/MCP 工具轮次上限（1–256）。见下方行为变更说明。 | +| `tool_timeout_seconds` | `120.0` | 单个 ADK/MCP 工具调用的超时秒数；设为 `None` 表示不超时。 | +| `reuse_workspace` | `False` | 仅在同时设置了 `workspace_root` 时生效，而且只值得在单租户场景下开启，见上方 [Workspace 生命周期](#workspace-生命周期)。否则两个字段都别设，让工具调用 `current_workspace()`，见 [Workspace 是数据面](#workspace-是数据面)。 | + + +**行为变更：`max_tool_iterations` 的含义已改变，默认值从 `8` 提高到 `32`。** +它现在约束的是**整个 Codex turn**，而不再是单次后端请求。Codex 每完成一次原生工具调用就会发起一次新的后端请求，所以旧的「每请求」计数实际允许的执行次数是「请求轮数 × budget」。默认值同时调高，是为了让「先跑若干轮原生工具、之后才调用 ADK 工具」的回合不被提前截断。如果你此前靠 `8` 这个值来兜底控制成本，请重新评估——现在更合适的成本上限是 `RunConfig(max_llm_calls=...)`（见下）。 + + +### 指令是怎么下发给 Codex 的 + +Codex 自带一份约 20KB、针对自身工具链（`apply_patch`、`update_plan`、shell、AGENTS.md）调优过的系统提示词。VeADK **不会**通过 `base_instructions` 覆盖它——Codex 在该字段被设置时会**整体替换**内置模板，这些指导会被全部删掉（Codex 官方文档也强烈不建议这么做）。 + +因此，Agent 的身份块与 `instruction` 走的是 Codex 原生的 `developer_instructions` 通道：Codex 把它渲染成一条独立的 `developer` 消息，与 AGENTS.md、skills、环境上下文并列，是纯追加而非替换。 + +副作用是 **`personality` 现在真的会生效**：它渲染在 Codex 内置的系统提示词模板里，而以前只要设置了 `base_instructions`，整个模板连同 personality 一起被替换掉，该字段形同虚设。 + +同一条通道上还会追加两条更正——Codex 被保留下来的系统提示词描述的工具链，这座桥并不能完整提供。runtime 会在每一轮的 developer instructions 后面追加一段简短的**工具可用性说明**： + +- 本次运行没有 `apply_patch`：shim 只转发 `function` 类型的工具，而 Codex 的文件编辑工具不是，所以创建和修改文件要用 `exec_command`（例如 `cat > file <<'EOF'` heredoc）。 +- `request_user_input` 确实被通告了，但没有人能回答它：一次 ADK 调用没有交互通道，调用它只会让这一轮什么都没做就结束。说明里会要求模型用手上已有的信息作判断，并在最终回复里讲清楚缺了什么。 + +这两点你不需要再在自己 Agent 的 `instruction` 里手写一遍。 + +### Codex 环境变量 + + +以下四个环境变量**覆盖** Python 中的 `CodexRuntimeConfig`，而不是作为它的默认值。也就是说，即使代码里写死了 `sandbox="workspace_write"`，部署环境里的 `VEADK_CODEX_SANDBOX=full_access` 依然会生效。这一优先级与大多数配置项相反，在容器/函数计算等由平台注入环境变量的场景中尤其需要注意。 + + +| 环境变量 | 覆盖的配置项 | 说明 | +| :--- | :--- | :--- | +| `VEADK_CODEX_SANDBOX` | `sandbox` | 取值同 `sandbox` 字段。 | +| `VEADK_CODEX_APPROVAL_MODE` | `approval_mode` | 取值同 `approval_mode` 字段。 | +| `VEADK_CODEX_WORKSPACE_ROOT` | `workspace_root` | 工作目录绝对路径。 | +| `VEADK_CODEX_NETWORK_ACCESS` | `network_access` | `1`/`true`/`yes`/`on` 视为开启，其余视为关闭。 | + +另有四个用于 Responses→Chat 转换层（shim）的环境变量。它们不参与上面的覆盖规则，只是 shim 自身的调参： + +| 环境变量 | 默认值 | 说明 | +| :--- | :--- | :--- | +| `CODEX_SHIM_NUM_RETRIES` | `2` | 后端请求遇到 429/5xx/超时等瞬时错误时的重试次数。 | +| `CODEX_SHIM_TIMEOUT` | `0`（不超时） | 单次后端请求的超时秒数。 | +| `CODEX_SHIM_START_TIMEOUT` | `10` | 等待 shim 的本地 HTTP 服务起来的秒数；超时即判定启动失败并明确报错，而不是让 Codex 一直连不上。 | +| `CODEX_SHIM_CACHE_MAX` | `8` | 进程内缓存的 shim 实例上限（按后端地址+凭证区分），用于限制多租户进程里的服务与端口数量。 | + +### Codex 可观测性 + +Codex 原生生命周期和 ADK Function/MCP 工具调用都会转换为 ADK Event。运行日志使用稳定的 `codex_*` 事件名，并包含 `invocation_id`、`call_id`、`tool`、`status`、`duration_ms` 等可归因字段。日志不会记录工具参数、工具结果、API Token、凭证或后端地址；Token Usage 通过 `codex_event_type=token_usage` 事件及对应日志提供。 + +此外： + +- **`call_llm` span**：ADK 的 `call_llm` span 本由它自己的 LLM flow 打开，而该 flow 已被 codex 替换，因此 runtime 会自行打开一个同名 span，并在回合结束时写入 prompt、响应与 token 用量。VeADK 的整条遥测链路——内存 exporter 的 Session 索引、Trace 上报、Portal 指标、以及基于 Trace 的评测——因此都能看到 codex 回合。注意粒度：**每回合一个 span**，而不是每次内层模型调用一个。 +- **`usage_metadata`**：整轮的 token 用量汇总后，只挂在**每回合一个的合并后终态 Event** 上（而不是逐条事件累加），避免下游做求和统计时重复计数。 +- **`RunConfig(max_llm_calls=...)` 会被强制执行**：shim 在每一次真实的后端模型调用前扣减配额。超限时 Codex 侧收到一个 `429 llm_calls_limit`（而不是它会重试的 500——重试会把本回合的工具副作用重跑一遍），回合结束后 runtime 再向调用方重新抛出 `LlmCallsLimitExceededError`，而不是返回 Codex 勉强拼出的半截答案。 + +## 支持矩阵 + +`runtime="codex"` 和 `runtime="piagent"` 会替换整个 ADK LLM 流程，因此 `Agent` 上相当一部分配置不会生效。VeADK 会在智能体构造时以及每次调用前检查这些配置：**产生错误结果的配置直接报错（`ValueError`）**，**只是被忽略的配置记录一次警告**。 + +### 报错：直接拒绝运行 + +| 配置 | 原因 | +| :--- | :--- | +| `model=...` | 运行时从 `model_name` 解析模型，完全忽略 `model` 对象，其 `api_base`、请求头和 fallback 都会丢失。请改用 `model_name`。 | +| `generate_content_config=...` | 只有 `system_instruction` 会被转发，`temperature`、`max_output_tokens`、`thinking_config` 等都会被丢弃。 | +| `output_schema=...` | schema 既不会下发给后端，也不会进入提示词，模型从未被要求按 schema 输出；`state[output_key]` 要么是未校验的回复，要么直接缺失。 | +| `planner=...` / `code_executor=...` | 二者以 ADK 请求/响应处理器的形式运行，外部运行时不执行这一层。 | +| `include_contents="none"` | 外部运行时始终发送完整会话历史，该设置会被静默忽略并泄漏历史轮次。 | +| `enable_supervisor=True` | 监督流程挂在 ADK LLM flow 上，而该流程已被替换，不会有任何监督生效。 | + + +`sub_agents` 会通过 ADK 兼容的 `transfer_to_agent` 桥接支持。外部运行时可以请求转交，VeADK 再在外层 ADK 上下文中运行被选中的目标智能体。 + + +### 警告：被忽略但仍可运行 + +| 配置 | 行为 | +| :--- | :--- | +| `model_name=[主, 备...]` | 只使用第一个模型，fallback 链不生效。 | +| `model_provider`（非 `openai`） | 运行时始终以 OpenAI 兼容协议访问 `model_api_base`。 | +| `model_extra_config` | 仅对 `piagent` 成立。**codex 会转发它**：`extra_headers` 与 `extra_body`（含 VeADK 默认的请求加密与 prompt 缓存配置）由 shim 原样带到后端请求上，与 `adk` 路径一致。 | +| `enable_responses` / `enable_responses_cache` | 不使用 Ark Responses API，`previous_response_id` 续接与响应缓存均不生效。 | +| `example_store` | 由 `ExampleTool.process_llm_request` 注入，外部运行时不调用该钩子，few-shot 示例不会进入 prompt。 | +| `knowledgebase` | **知识库被静默禁用。** 同样由 `LoadKnowledgebaseTool.process_llm_request` 注入使用说明，外部运行时不调用，模型不知道知识库的存在，检索不会被触发。 | +| `skills_mode` | 外部运行时在桥接工具时会跳过 VeADK 的 `SkillsToolset`，指令里宣称的 `execute_skills`/`skills_tool` 并未注册。 | +| `enable_skills_checklist` | 依赖 ADK 的 before-tool 回调与 skills 工具集，两者都不生效。 | +| `after_model_callback` | 语义不同：每回合在合并后的最终文本上触发一次，而不是每次模型调用触发一次。 | +| `tracers` | 粒度不同：codex 会补一个**每回合一个**的 `call_llm` span（含 prompt、响应、整轮 token 用量），但内层模型循环跑在外部 harness 里，仍然没有逐次模型调用的 span。 | +| `codex_runtime_config`（配合 `runtime="piagent"`） | 只对 codex 运行时有效，piagent 完全忽略。 | +| `RunConfig(max_llm_calls=...)` | 仅对 `piagent` 成立——它的模型循环在 harness 内部自行计数，不受该配额约束。**codex 已支持**：见上方[可观测性](#codex-可观测性)。 | + +### 不支持 `run_live` + +非 `adk` 运行时没有 live/bidi 实现。通过 ADK `get_fast_api_app` 暴露的 `/run_live`（`veadk web` 使用）对这类智能体会抛出 `NotImplementedError`，而不是静默回退到 ADK 流程——否则会用完全不同的模型循环和工具集执行。live 场景请使用 `runtime="adk"`，其余场景使用 `runner.run_async`。 ## 请求处理 diff --git a/examples/README.md b/examples/README.md index 69586c78e..e7b9533be 100644 --- a/examples/README.md +++ b/examples/README.md @@ -37,6 +37,25 @@ The examples are grouped by concept: 01–02 basics, 03 & 09 memory, 04–05 too knowledge, 06 & 10 multi-agent, 07–08 model behavior, 11 observability, and 13 OpenViking-backed knowledge and memory. +## Codex runtime + +Four examples use `Agent(runtime="codex")`, which hands the inner loop to a +sandboxed coding agent that can write a file, run it, read the traceback and fix +it. Two of them show **what the runtime is for**; two show **how to wire it up**. + +| Example | Kind | What you'll learn | +| --- | --- | --- | +| [Data analysis](./codex_data_analysis/) | What it's for | Codex writes an analysis script, hits a real error in dirty data, fixes it, re-runs, reports — the self-iteration loop | +| [Ops assistant](./codex_ops_assistant/) | What it's for | Correlate logs, metrics and deploys with throwaway scripts to find a root cause, inside a no-network sandbox | +| [Skill + MCP](./codex_with_skill_and_mcp/) | How to wire it | The paths a local skill and an MCP tool take under this runtime | +| [Deploy to AgentKit](./codex_runtime_on_agentkit/) | How to deploy | Ship a `runtime="codex"` agent to Volcengine AgentKit | + +Reach for this runtime when the steps cannot be enumerated as tools ahead of +time — ad-hoc analysis, log triage, data wrangling. Keep `runtime="adk"` for a +fixed tool call and a formatted answer: it is faster, cheaper, and does not +reject `sub_agents` / `output_schema` / `planner` / `code_executor`. See +[when to use the codex runtime](../docs/content/docs/framework/agent/runtime.en.mdx#when-to-use-the-codex-runtime). + ## Common setup 1. Install VeADK (example 05 with the local backend needs the `extensions` extra): diff --git a/examples/README.zh.md b/examples/README.zh.md index f3dbfab07..e176eeafe 100644 --- a/examples/README.zh.md +++ b/examples/README.zh.md @@ -34,6 +34,24 @@ 这些示例按概念分组：01–02 基础，03 与 09 记忆，04–05 工具与知识， 06 与 10 多智能体，07–08 模型行为，11 可观测性，13 为 OpenViking 知识与记忆。 +## Codex 运行时 + +有四个示例使用 `Agent(runtime="codex")`——它把内层循环交给一个沙箱里的 coding +agent，让模型可以写下文件、跑起来、读到报错、再改掉。其中两个讲**这个运行时是干 +什么用的**，另外两个讲**怎么把它接起来**。 + +| 示例 | 类型 | 你将学到 | +| --- | --- | --- | +| [数据分析](./codex_data_analysis/) | 用来干什么 | Codex 写分析脚本、在脏数据上撞到真实报错、自己改好、重跑、出报告——自迭代循环 | +| [运维助手](./codex_ops_assistant/) | 用来干什么 | 用一次性脚本把日志、指标与发布记录关联起来定位根因，全程在断网沙箱里 | +| [Skill + MCP](./codex_with_skill_and_mcp/) | 怎么接线 | 本地 skill 与 MCP 工具在这个运行时下分别走哪条路 | +| [部署到 AgentKit](./codex_runtime_on_agentkit/) | 怎么部署 | 把一个 `runtime="codex"` 智能体发布到火山引擎 AgentKit | + +当任务的步骤无法事先枚举成工具时（临时分析、日志排障、数据清洗）才用这个运行时。 +如果只是一次固定的工具调用加一段格式化回答，请继续用 `runtime="adk"`：更快、更便宜， +而且不会拒绝 `sub_agents` / `output_schema` / `planner` / `code_executor`。 +详见[什么时候该用 codex 运行时](../docs/content/docs/framework/agent/runtime.mdx#什么时候该用-codex-运行时)。 + ## 通用准备 1. 安装 VeADK（示例 05 使用 local 后端时需要 `extensions` 扩展）： diff --git a/examples/codex_data_analysis/.env.example b/examples/codex_data_analysis/.env.example new file mode 100644 index 000000000..1adcbaea2 --- /dev/null +++ b/examples/codex_data_analysis/.env.example @@ -0,0 +1,11 @@ +# Ark (or any OpenAI-compatible chat) credentials. +MODEL_AGENT_API_KEY=your-ark-api-key +MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 + +# Model choice matters here — see "Known rough edges" in the README. +# Verified end to end; doubao-seed-1-6-250615 cannot complete a multi-round turn. +MODEL_AGENT_NAME=deepseek-v4-flash-260425 + +# Required: Ark rejects VeADK's default prompt caching alongside the +# `instructions` field Codex always sends. +MODEL_AGENT_CACHING=disabled diff --git a/examples/codex_data_analysis/.gitignore b/examples/codex_data_analysis/.gitignore new file mode 100644 index 000000000..0b8c697e9 --- /dev/null +++ b/examples/codex_data_analysis/.gitignore @@ -0,0 +1,5 @@ +# Generated at run time. Codex's workspace is not here -- the runtime keeps +# it under its own per-session temp root, which `current_workspace()` +# resolves for the tools. +outbox/ +__pycache__/ diff --git a/examples/codex_data_analysis/README.md b/examples/codex_data_analysis/README.md new file mode 100644 index 000000000..5f37f6606 --- /dev/null +++ b/examples/codex_data_analysis/README.md @@ -0,0 +1,344 @@ +# codex_data_analysis + +An agent that turns a raw sales extract into a published report **by writing an +analysis script, running it, reading the traceback, fixing it, and running it +again** — inside an OS sandbox with the network switched off. + +> 中文版见 [README.zh.md](./README.zh.md) + +This is the flagship `runtime="codex"` example. If you want the wiring reference +for skills and MCP tools under this runtime, see +[`codex_with_skill_and_mcp/`](../codex_with_skill_and_mcp/) instead. + +``` +codex_data_analysis/ +├── main.py # the agent, its sandbox settings, and a 2-turn run +├── analytics_tools.py # the two ADK tools that bracket the sandbox +├── data/ +│ └── sales_2025q3.csv # the "internal system": 2 400 raw orders, defects included +├── skills/ +│ └── sales-report/ +│ └── SKILL.md # the house format for the report +└── outbox/ # created at run time — what publish_report let out +``` + +## The task + +1. `fetch_sales_extract` (ADK tool) exports one quarter of orders from the + internal warehouse **into Codex's workspace** and returns a receipt: + `{"path": "data/sales_2025q3.csv", "rows": 2400, ...}`. +2. **Codex** writes an analysis script, runs it, hits a real `ValueError`, fixes + it, re-runs, and produces `report.md` + a hand-written `chart.svg`. +3. `publish_report` (ADK tool) validates the paths and copies both files into + `outbox/` — the only way anything leaves the sandbox. + +Then a second turn — *"replace the trend chart with revenue by region"* — +reuses the same workspace: the extract, the script and the report are still +there, so the agent edits rather than starting over. + +## Why this task suits codex, and not adk + +The honest comparison is not "can the model run code" — `runtime="adk"` has +code executors too. It is **what shape of loop you get, and what it costs to +run it safely.** + +**What this runtime gives you here:** + +- **A working directory and a shell, not an expression evaluator.** Debugging is + a loop of `ls`, `cat`, write file, `python3 x.py`, read traceback, patch, + re-run. ADK's code-executor path evaluates a *block the model emits* and hands + back its output; it is not a place the model can accumulate a script, a data + file, a chart and a report next to each other and iterate over them. +- **An OS sandbox with no infrastructure to provision.** macOS seatbelt / Linux + landlock+seccomp, established by the Codex CLI itself. ADK ships + `UnsafeLocalCodeExecutor` — which executes in *your* process, with no + isolation and not even stateful — and `BuiltInCodeExecutor`, which delegates + to the model provider's server-side tool and is Gemini-only, so it is not + available on an Ark chat backend at all. Anything safer, you provision + yourself. Here `sandbox="workspace_write"` + `network_access=False` is four + lines of config. +- **Files that survive the turn.** The workspace is session-scoped, so turn 2 + amends turn 1's artifacts. A code-executor block starts from nothing each time + unless you rebuild the state yourself. +- **A debugging harness you don't have to prompt into existence.** Reading a + traceback and patching the file is what Codex's own loop already does. + +**What you pay for it:** + +- A Codex subprocess per turn, and one backend request per native tool round, + each re-serializing the turn. This example takes minutes and tens of model + calls where a one-shot answer takes one. +- A large part of the `Agent` surface is refused outright (see + [Constraints](#constraints)) and per-LLM-call callbacks never run. +- Non-determinism in the number of rounds. Budget for it with + `RunConfig(max_llm_calls=...)`. + +**When you should *not* reach for it:** + +- One tool call and a formatted answer. `runtime="adk"` does that faster and + cheaper — a Codex subprocess buys you nothing. +- You need `output_schema`, `sub_agents`, a `planner`, or per-call model + callbacks. All refused under this runtime. +- The computation is known in advance. If you already know the analysis is + "group by region, sum revenue", write that in Python and expose it as an ADK + tool. This runtime earns its cost when **the code is not knowable ahead of + time** — one-off analyses, unfamiliar file formats, data whose defects you + discover only by running against it. +- Latency-sensitive interactive chat. + +## The one rule to get right: paths, not payloads + +> **The workspace is the data plane. Tool arguments and results are the control +> plane.** + +Under this runtime an ADK tool is executed by the runtime's Responses shim, and +its JSON result is fed back to the model **as text in its context**. It does not +land in a file. So: + +```python +# WRONG — 2 400 rows enter the context on this request and every later request +def fetch_sales_extract(quarter: str) -> dict: + return {"rows": [...]} # ~125 KB of CSV, re-sent on every round + +# RIGHT — the data goes to disk, the receipt goes to the model +def fetch_sales_extract(quarter: str) -> dict: + workspace = Path(current_workspace()) # this turn's sandbox directory + shutil.copyfile(source, workspace / "data" / source.name) + return {"status": "ok", "path": "data/sales_2025q3.csv", "rows": 2400, + "columns": [...], "bytes": 127983} +``` + +The model then reads the file with its own sandboxed code, where volume is free. +`publish_report` is the same rule in reverse: it takes *paths* and returns a +receipt of what it copied out. + +The extract's size is doing real work here. At 40 rows a model just `cat`s the +file, sees every defect, and writes a correct script first try — the debugging +loop never happens and the example proves nothing. At 2 400 rows `cat` is +useless (Codex truncates command output), so the only way to learn what is in +the file is to write code against it and see what breaks. That is what the +runtime is for, and it is also what real extracts look like. + +Two consequences worth internalising: + +- **Tool docstrings should say so.** `fetch_sales_extract`'s docstring tells the + model "returns a receipt, not the data — read the CSV at the returned path". + Without that, models try to use the receipt as the data. +- **Paths from the model are untrusted input.** `publish_report` resolves every + path against the workspace and rejects anything that escapes it (`..`, + absolute paths, symlinks out). See `_resolve_in_workspace`. + +### How the tools know where the workspace is + +The ADK tools run in *your* process, not in the sandbox, so they have to be +told where Codex is working. They ask, once per call: + +```python +from veadk.runtime.codex import current_workspace + +def fetch_sales_extract(quarter: str) -> dict: + workspace = current_workspace() # this turn's directory, or None + if workspace is None: # not a codex turn — say so, don't guess + return {"status": "error", "message": "no sandbox working directory"} + ... +``` + +The runtime binds that value around each tool call, so it is *this* turn's +workspace even with several sessions in flight in one process. Which is why the +example leaves `workspace_root` and `reuse_workspace` unset: each +`(app, user, session, agent)` gets its own directory, and it still persists +across the turns of that session — the property turn 2 relies on. + +`current_workspace()` returns `None` rather than raising when no codex turn is +on the stack (another runtime, an `AgentTool`, a unit test). The tools here turn +that into an ordinary `{"status": "error", ...}` result the model can read, +instead of raising or quietly falling back to a directory of their own. + +**Pinning is the single-tenant convenience, not the multi-tenant answer.** +`workspace_root=..., reuse_workspace=True` makes the directory a constant you +can `ls` long after the process exits — useful while developing one agent on +your own machine, wrong on a server, where it collapses every session onto one +directory. Unpinned workspaces live under a temporary root the runtime owns and +are removed when the process exits, which is why `main.py` prints the tree +before it finishes. + +## Security is the demo, not boilerplate + +```python +CodexRuntimeConfig( + sandbox="workspace_write", # may write only in its own workspace + network_access=False, # honoured by workspace_write: no sockets + approval_mode="deny_all", # refuse every escalation Codex asks for + max_tool_iterations=8, # ADK tool round-trips for the whole turn +) +... +run_config = RunConfig(max_llm_calls=60) # hard cost ceiling +``` + +The story these four lines tell: + +- The model may compute **anything** over the data — that is the point — but + with the network off it has no socket to send it through. `outbox/` sits + *outside* the workspace, so the sandbox cannot write there either. +- That leaves the two audited ADK tools as the **only** outbound path. Every + file that leaves went through `publish_report`, which logs it, size-limits it, + digests it, and refuses anything that is not a `.md` or `.svg` from inside the + workspace. That is a boundary you can point an auditor at. +- `approval_mode="deny_all"` keeps it that way. **Never use `"auto_review"`** in + an example or a deployment: despite the name it is *full auto-approval* — the + Codex SDK's built-in handler accepts every escalation and cannot be replaced. +- `max_tool_iterations` bounds ADK tool round-trips for the **whole turn** + (default 32). It is not a cost ceiling; `RunConfig(max_llm_calls=...)` is. The + codex runtime is the one external runtime that enforces that budget exactly — + the shim charges it *before* each backend call. + +## Why the data is dirty + +`data/sales_2025q3.csv` is a realistic warehouse export, which means it is a +mess in four ordinary ways: + +| Defect | Rows | What a naive script does | +| --- | --- | --- | +| Thousands separators in a numeric column (`"4,208.40"`) | 22 | `ValueError: could not convert string to float: '4,208.40'` | +| A blank revenue cell | 4 | `ValueError: could not convert string to float: ''` | +| A second date format (`07/23/2025`) | 31 | `ValueError: time data '07/23/2025' does not match format '%Y-%m-%d'` | +| Three spellings of one region (`north`, `NORTH`, `_South`) | 14 | no crash — silently splits one region into three | + +**The first defect is on line 212**, so `head` looks perfectly clean and the +first script is written against a file that seems fine. + +**Nothing in the prompt mentions any of this.** That is deliberate: if the +instruction described the defects, the model would write a defensive parser on +the first try and the example would demonstrate nothing that `runtime="adk"` +could not do. The first script has to actually crash for the loop to be real. + +The last row is the interesting one — it doesn't crash, so only an agent that +*looks at its own output* catches it. That is what the report's `Data notes` +section is for. + +## The skill + +`skills/sales-report/SKILL.md` carries the house report format (sections, +column order, money formatting, hand-written-SVG rules). It is loaded the +ADK-native way and materialised into Codex's own skill directory, so Codex's +native skill system discovers and progressively loads it. Two reasons it earns +its place here: the format stays out of the prompt, and its `Data notes` +section is what makes the agent's iteration visible in the finished artifact. + +## Run + +```bash +pip install "veadk-python[codex]" # openai-codex + the bundled Codex CLI binary + +export MODEL_AGENT_API_KEY=... +export MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 +export MODEL_AGENT_NAME=deepseek-v4-flash-260425 # see Known rough edges: model choice matters + +# Known issue: Ark rejects the request when VeADK's default prompt caching is +# combined with the `instructions` field Codex always sends +# ("caching is not supported for instructions"). Disable it for now: +export MODEL_AGENT_CACHING=disabled + +python examples/codex_data_analysis/main.py +``` + +macOS or Linux only — the sandbox is seatbelt / landlock+seccomp. + +### What to watch for + +`main.py` prints every sandboxed command as it runs, because the runtime +surfaces Codex's own `commandExecution` items as ordinary ADK function-call +events: + +``` + → fetch_sales_extract({'quarter': '2025Q3'}) + ← fetch_sales_extract: {'status': 'ok', 'path': 'data/sales_2025q3.csv', 'rows': 2400, ...} + $ cat .../skills/sales-report/SKILL.md + $ head -20 data/sales_2025q3.csv # looks perfectly clean + $ cat > analyze.py << 'PY' ... PY; python3 analyze.py + exit=1 + | ValueError: could not convert string to float: '' + $ cat > analyze.py << 'PY' ... PY # rewritten + $ python3 -c "... find the bad rows ..." + $ cat > build_report.py << 'PY' ... PY; python3 build_report.py + $ cat report.md + → publish_report({'report_path': 'report.md', 'chart_path': 'chart.svg'}) + ← publish_report: {'status': 'ok', 'published': [...]} +``` + +That is a real transcript, lightly abridged: 12 sandboxed commands, two of them +failing, `head -20` showing nothing wrong because the first defect is 190 lines +further down. Turn 2 then read the report and chart it had left behind, replaced +just the chart and the Trend paragraph, recovered from a `zsh` quoting error, +and republished. Expect that shape rather than those exact commands — the number +of rounds varies from run to run. + +The run ends by printing what Codex left in its workspace — its script, its +drafts — because that directory belongs to the session and the runtime removes +it when the process exits. What stays on disk is the outbox: + +```bash +cat examples/codex_data_analysis/outbox/*/report.md +``` + +The `Data notes` section of the report lists the defects it had to work around +— compare it against the table above to see how much it caught. In the run +above it reported three of the four classes with exact row counts (4 null +revenues, 31 mis-formatted dates, 14 region-casing rows) and quietly handled +the fourth; every published figure matched the ground truth to the cent. + +## Known rough edges (as of this writing) + +These are runtime/backend issues, not example bugs. They shape the code above, +so they are worth knowing before you build on it. + +- **Ark rejects VeADK's default prompt caching under this runtime.** Codex + always sends the Responses `instructions` field, and Ark answers + `400 InvalidParameter: caching is not supported for instructions`. Every + codex-runtime agent on Ark fails on the first backend call until you set + `MODEL_AGENT_CACHING=disabled`. +- **Not every Ark model can be the backend, and the failure is silent.** After + its first tool round Codex replays its own `reasoning` items in the + conversation, and the shim forwards them verbatim. `doubao-seed-1-6-250615` + answers `400 InvalidParameter: input[N].reasoning ... Item reasoning is not + supported for model`. What you see is *not* an error: the first backend call + succeeds, the agent runs exactly one command, and the turn ends + `status=completed` with a half-finished workspace and a cheerful summary. The + 400 appears only as a `codex_backend_api_error` warning in the log — nothing + is raised to the caller. Use `deepseek-v4-flash-260425` (verified end to end), + and when a codex turn stops suspiciously early, grep the log for + `codex_backend_api_error` before believing the answer. +- **A chat model bridged into Codex's protocol narrates instead of acting.** + Codex ends a turn on an assistant message, so a model that replies *"I'll now + write the analysis script"* silently ends the turn with nothing done. That is + why the instruction opens with *"Act, do not narrate"*. Expect to spend prompt + budget on this with any chat backend. +- **`apply_patch` never reaches the backend, and nobody can answer + `request_user_input`.** The shim forwards only `function`-typed tools, and + Codex's file-editing tool is not one — the list the backend actually sees is + `exec_command`, `write_stdin`, `update_plan`, `request_user_input`, + `view_image`, plus your ADK tools. Codex's own system prompt still tells the + model to use `apply_patch`, and `request_user_input` is advertised even though + an ADK invocation has no interactive channel to answer it on. **The runtime + appends a tool-availability note to every turn's developer instructions** + stating both facts and what to do instead — create files with an + `exec_command` heredoc, decide rather than ask. This example's instruction + used to carry those two sentences by hand and no longer needs to. + +The general lesson: on a chat backend this runtime's effective tool surface is +narrower than Codex's documentation implies. The runtime closes those two gaps +for you; anything else is still your instruction's job. + +## Constraints + +`runtime="codex"` refuses a large part of the `Agent` surface rather than +silently ignoring it. Relevant here: + +- rejected: `sub_agents`, `model=` (use `model_name=`), `output_schema`, + `planner`, `code_executor`, `include_contents="none"`, `enable_supervisor`, + and any `generate_content_config` field other than `system_instruction`; +- also rejected: `CodexRuntimeConfig(sandbox="full_access", network_access=False)` + — that combination reads as "no network" while granting full access; +- dropped with a warning: `knowledgebase`, `example_store`, `skills_mode`, ... + +See the [support matrix](../../docs/content/docs/framework/agent/runtime.en.mdx#support-matrix). diff --git a/examples/codex_data_analysis/README.zh.md b/examples/codex_data_analysis/README.zh.md new file mode 100644 index 000000000..200556ae9 --- /dev/null +++ b/examples/codex_data_analysis/README.zh.md @@ -0,0 +1,292 @@ +# codex_data_analysis + +一个把原始销售流水变成正式报告的 Agent——**它自己写分析脚本、运行、读报错、改脚本、再运行**， +全过程都在操作系统级沙箱里，且网络已关闭。 + +> English version: [README.md](./README.md) + +这是 `runtime="codex"` 的旗舰示例。如果你要找的是 skill / MCP 工具在该 runtime 下的接线参考， +请看 [`codex_with_skill_and_mcp/`](../codex_with_skill_and_mcp/)。 + +``` +codex_data_analysis/ +├── main.py # Agent、沙箱配置，以及一次两轮对话 +├── analytics_tools.py # 夹在沙箱两端的两个 ADK 工具 +├── data/ +│ └── sales_2025q3.csv # “内部系统”：2400 条原始订单，脏数据俱全 +├── skills/ +│ └── sales-report/ +│ └── SKILL.md # 报告的固定格式（house format） +└── outbox/ # 运行时创建——只有 publish_report 放行的文件 +``` + +## 这个任务做什么 + +1. `fetch_sales_extract`（ADK 工具）把某个季度的订单从内部数仓导出**到 Codex 的工作区**， + 只返回一张回执：`{"path": "data/sales_2025q3.csv", "rows": 2400, ...}`。 +2. **Codex** 写一个分析脚本、运行、撞上真实的 `ValueError`、修好、再运行， + 最终产出 `report.md` 和一张手写的 `chart.svg`。 +3. `publish_report`（ADK 工具）校验路径，把两个文件复制进 `outbox/`——这是任何文件离开沙箱的唯一出口。 + +随后的第二轮——*“把趋势图换成按地区的收入排行”*——复用同一个工作区：数据、脚本、报告都还在， +所以 Agent 是去改，而不是从头再来。 + +## 为什么这个任务适合 codex，而不适合 adk + +真正该比的不是“模型能不能跑代码”——`runtime="adk"` 也有 code executor。 +该比的是 **你拿到的是什么形状的循环，以及安全地跑起来要付出什么代价。** + +**这个 runtime 在这里给你的东西：** + +- **一个工作目录加一个 shell，而不是一个表达式求值器。** 调试就是 `ls`、`cat`、写文件、 + `python3 x.py`、读 traceback、打补丁、再跑一遍这样一个循环。ADK 的 code executor 是把 + *模型吐出的代码块*执行掉再把输出还回去；它不是一个能让模型把脚本、数据文件、图表、报告 + 堆在一起反复迭代的地方。 +- **一个不需要额外基础设施的操作系统沙箱。** macOS seatbelt / Linux landlock+seccomp，由 + Codex CLI 自己建立。ADK 自带的是 `UnsafeLocalCodeExecutor`——直接在*你的*进程里执行， + 没有任何隔离，甚至不支持有状态——以及 `BuiltInCodeExecutor`，后者转交给模型厂商的服务端工具， + 只支持 Gemini，在 Ark 聊天后端上根本用不了。更安全的方案都得你自己搭。 + 而这里 `sandbox="workspace_write"` + `network_access=False` 就是四行配置。 +- **能跨轮次存活的文件。** 工作区按会话隔离，所以第二轮是在第一轮的产物上继续改。 + code executor 的代码块每次都从零开始，除非你自己把状态重建出来。 +- **一套不用靠提示词拼出来的调试机制。** 读 traceback、改文件重跑，本来就是 Codex 自身循环在做的事。 + +**你要为此付出的代价：** + +- 每一轮都要拉起一个 Codex 子进程，而且每个原生工具回合都要发一次后端请求， + 每次都把整轮上下文重新序列化一遍。这个示例要跑几分钟、几十次模型调用， + 而一次性回答只要一次。 +- `Agent` 的相当一部分配置面会被直接拒绝（见 [约束](#约束)）， + 并且 per-LLM-call 回调完全不会执行。 +- 回合数不确定。用 `RunConfig(max_llm_calls=...)` 给它兜底。 + +**什么时候*不该*用它：** + +- 一次工具调用加一句格式化回答。`runtime="adk"` 更快更便宜——为此拉起 Codex 子进程毫无收益。 +- 你需要 `output_schema`、`sub_agents`、`planner` 或 per-call 模型回调。这些在该 runtime 下都被拒绝。 +- 计算逻辑事先就已知。如果你早就知道分析就是“按地区分组求和”，那就直接用 Python 写好、 + 包成一个 ADK 工具。这个 runtime 值回票价的场景是 **代码事先无法确定**：一次性分析、 + 不熟悉的文件格式、只有真的跑一遍才会暴露的数据缺陷。 +- 对延迟敏感的交互式对话。 + +## 唯一必须做对的事：传路径，不传数据 + +> **工作区是数据面，工具的入参和返回值是控制面。** + +在该 runtime 下，ADK 工具由 runtime 的 Responses shim 执行，它的 JSON 返回值会 +**以文本形式回到模型上下文里**，而不会落到文件上。所以： + +```python +# 错 —— 2400 行数据会进入本次请求，并且此后每次请求都跟着走 +def fetch_sales_extract(quarter: str) -> dict: + return {"rows": [...]} # 约 125 KB 的 CSV，每一回合都重发一遍 + +# 对 —— 数据落盘，回执给模型 +def fetch_sales_extract(quarter: str) -> dict: + workspace = Path(current_workspace()) # 本轮沙箱的工作目录 + shutil.copyfile(source, workspace / "data" / source.name) + return {"status": "ok", "path": "data/sales_2025q3.csv", "rows": 2400, + "columns": [...], "bytes": 127983} +``` + +模型随后用自己的沙箱代码去读这个文件，那里的数据量是免费的。 +`publish_report` 是同一条规则的反方向：入参是*路径*，返回的是“复制出去了什么”的回执。 + +这份数据的**体量本身就是设计的一部分**。只有 40 行时，模型直接 `cat` 一下就看全了所有脏数据， +第一版脚本就写对——调试循环根本不会发生，示例也就什么都证明不了。到了 2400 行， +`cat` 没有用（Codex 会截断命令输出），要弄清文件里到底有什么，唯一的办法就是写代码跑一遍看哪里炸。 +这正是这个 runtime 存在的意义，也正是真实数据导出的样子。 + +有两点值得记住： + +- **工具的 docstring 要把这件事说清楚。** `fetch_sales_extract` 的 docstring 明确告诉模型 + “返回的是回执不是数据——请到返回的路径去读 CSV”。不写这句，模型会试图把回执当数据用。 +- **模型给的路径是不可信输入。** `publish_report` 会把每个路径解析回工作区， + 并拒绝任何越界的路径（`..`、绝对路径、指向外部的符号链接）。见 `_resolve_in_workspace`。 + +### 工具是怎么知道工作区在哪的 + +ADK 工具跑在*你的*进程里，而不是沙箱里，所以必须有人告诉它们 Codex 在哪工作。 +它们每次被调用时自己问一遍： + +```python +from veadk.runtime.codex import current_workspace + +def fetch_sales_extract(quarter: str) -> dict: + workspace = current_workspace() # 本轮的工作目录，或者 None + if workspace is None: # 不在 codex 轮次里——直说，别猜 + return {"status": "error", "message": "no sandbox working directory"} + ... +``` + +这个值由 runtime 在每次工具调用前后绑定，所以即使一个进程里同时跑着多个会话， +拿到的也一定是*本轮*的工作区。正因如此，这个示例把 `workspace_root` 和 `reuse_workspace` +都留空：每个 `(app, user, session, agent)` 各得一个目录，并且照样能跨该会话的多轮存活—— +第二轮依赖的正是这个特性。 + +当调用栈上没有 codex 轮次时（换了 runtime、被 `AgentTool` 调用、单元测试）， +`current_workspace()` 返回 `None` 而不是抛异常。这里的工具因此把它转成一条普通的 +`{"status": "error", ...}` 结果交给模型，而不是抛异常，也不是悄悄退回到自己的某个本地目录。 + +**钉死目录如今是单租户下的便利，而不是多租户的答案。** +`workspace_root=..., reuse_workspace=True` 让目录变成一个常量，进程退出很久之后你依然能 +`ls` 它——在自己机器上开发单个 Agent 时很好用，放到服务端就是错的：它会把所有会话压到同一个目录。 +不钉死时，工作区位于 runtime 自己的临时根目录下，进程退出即被清理， +所以 `main.py` 会在结束前先把目录树打印出来。 + +## 安全配置本身就是这个示例的内容 + +```python +CodexRuntimeConfig( + sandbox="workspace_write", # 只能写自己的工作区 + network_access=False, # 只有 workspace_write 会读它：沙箱内没有任何 socket + approval_mode="deny_all", # 拒绝 Codex 提出的一切提权 + max_tool_iterations=8, # 整轮允许的 ADK 工具往返次数 +) +... +run_config = RunConfig(max_llm_calls=60) # 硬性成本上限 +``` + +这四行讲的是一个完整的故事： + +- 模型可以对数据做**任何**计算——这正是我们要的——但网络关掉之后，它没有任何 socket 把结果送出去。 + `outbox/` 刻意放在工作区**之外**，所以沙箱也写不到那里。 +- 于是两个受审计的 ADK 工具成了**唯一**的出口。每一个离开沙箱的文件都经过 `publish_report`： + 有日志、有大小上限、有摘要，并且拒绝任何非工作区内的、非 `.md`/`.svg` 的文件。 + 这是一条你可以指给审计人员看的边界。 +- `approval_mode="deny_all"` 维持这条边界。**永远不要用 `"auto_review"`**： + 名字有迷惑性，它其实是*全自动批准*——Codex SDK 内置的审批处理器会接受一切提权请求，且无法替换。 +- `max_tool_iterations` 约束的是**整轮**的 ADK 工具往返次数（默认 32），它不是成本上限； + 成本上限是 `RunConfig(max_llm_calls=...)`。codex 是唯一能精确执行这个预算的外部 runtime—— + shim 会在每次后端调用**之前**扣减它。 + +## 为什么数据是脏的 + +`data/sales_2025q3.csv` 是一份真实感的数仓导出，也就是说它以四种最常见的方式一团糟： + +| 缺陷 | 行数 | 天真的脚本会怎样 | +| --- | --- | --- | +| 数值列里带千分位（`"4,208.40"`） | 22 | `ValueError: could not convert string to float: '4,208.40'` | +| 一个空的金额单元格 | 4 | `ValueError: could not convert string to float: ''` | +| 第二种日期格式（`07/23/2025`） | 31 | `ValueError: time data '07/23/2025' does not match format '%Y-%m-%d'` | +| 同一个地区的三种写法（`north`、`NORTH`、`_South`） | 14 | 不报错——悄悄把一个地区拆成三个 | + +**第一处缺陷出现在第 212 行**，所以 `head` 看上去一切正常， +第一版脚本是对着一个「看起来没问题」的文件写出来的。 + +**提示词里对此只字未提。** 这是刻意的：如果 instruction 把缺陷都描述清楚， +模型第一版就会写出防御性的解析器，那这个示例也就演示不出任何 `runtime="adk"` 做不到的东西了。 +第一版脚本必须真的崩，这个循环才是真的。 + +最后一行最有意思——它不会崩，所以只有*会去看自己输出*的 Agent 才抓得到。 +报告里的 `Data notes` 小节就是为此存在的。 + +## 关于那个 skill + +`skills/sales-report/SKILL.md` 承载报告的固定格式（章节、列顺序、金额格式、手写 SVG 的规则）。 +它以 ADK 原生方式加载，并被 materialize 进 Codex 自己的 skill 目录， +由 Codex 原生的 skill 系统发现和渐进加载。它在这里值得存在有两个理由： +格式不必占用提示词；而它的 `Data notes` 小节让 Agent 的迭代过程在最终产物里显形。 + +## 运行 + +```bash +pip install "veadk-python[codex]" # openai-codex + 自带的 Codex CLI 二进制 + +export MODEL_AGENT_API_KEY=... +export MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 +export MODEL_AGENT_NAME=deepseek-v4-flash-260425 # 见「已知的粗糙之处」：选哪个模型很关键 + +# 已知问题：VeADK 默认开启的 prompt caching 与 Codex 必带的 `instructions` 字段冲突， +# Ark 会返回 400（"caching is not supported for instructions"）。暂时关掉： +export MODEL_AGENT_CACHING=disabled + +python examples/codex_data_analysis/main.py +``` + +仅支持 macOS 与 Linux——沙箱依赖 seatbelt / landlock+seccomp。 + +### 该看什么 + +`main.py` 会把每一条沙箱内执行的命令打印出来，因为 runtime 把 Codex 自己的 +`commandExecution` 条目转成了普通的 ADK function-call 事件： + +``` + → fetch_sales_extract({'quarter': '2025Q3'}) + ← fetch_sales_extract: {'status': 'ok', 'path': 'data/sales_2025q3.csv', 'rows': 2400, ...} + $ cat .../skills/sales-report/SKILL.md + $ head -20 data/sales_2025q3.csv # 看上去一切正常 + $ cat > analyze.py << 'PY' ... PY; python3 analyze.py + exit=1 + | ValueError: could not convert string to float: '' + $ cat > analyze.py << 'PY' ... PY # 重写 + $ python3 -c "... 把出问题的行找出来 ..." + $ cat > build_report.py << 'PY' ... PY; python3 build_report.py + $ cat report.md + → publish_report({'report_path': 'report.md', 'chart_path': 'chart.svg'}) + ← publish_report: {'status': 'ok', 'published': [...]} +``` + +这是一段真实记录（略作删节）：12 条沙箱命令，其中 2 条失败；`head -20` 什么问题都看不出来， +因为第一处缺陷在再往下 190 行的位置。第二轮接着读了它自己留下的 report 和 chart， +只替换了图和 Trend 段落，并从一次 `zsh` 引号错误里恢复过来，然后重新发布。 +请把它当作一种「形状」而不是固定命令——回合数每次都不一样。 + +这次运行结束前会把 Codex 留在工作区里的东西（它的脚本、它的草稿）打印出来—— +那个目录属于本会话，进程退出时会被 runtime 清理。留在磁盘上的是 outbox： + +```bash +cat examples/codex_data_analysis/outbox/*/report.md +``` + +报告里的 `Data notes` 小节会列出它绕过的缺陷——和上面那张表对照一下，看它抓到了多少。 +在上面那次运行里，它报出了四类缺陷中的三类并给出了准确行数（4 行空金额、31 行日期格式不一致、 +14 行地区写法不一致），第四类则是默默处理掉了；所有发布出来的数字都和真实值分毫不差。 + +## 已知的粗糙之处（截至撰写时） + +以下都是 runtime / 后端的问题，不是这个示例的 bug。它们直接影响了上面的代码写法， +在你基于它开发之前值得先知道。 + +- **在该 runtime 下 Ark 会拒绝 VeADK 默认开启的 prompt caching。** Codex 总会带上 + Responses 的 `instructions` 字段，而 Ark 返回 + `400 InvalidParameter: caching is not supported for instructions`。 + 在你设置 `MODEL_AGENT_CACHING=disabled` 之前，任何跑在 Ark 上的 codex-runtime Agent + 都会在第一次后端调用时失败。 +- **不是每个 Ark 模型都能当后端，而且失败是静默的。** 第一个工具回合之后， + Codex 会把自己的 `reasoning` 条目重放进对话，shim 原样转发给后端。 + `doubao-seed-1-6-250615` 会返回 + `400 InvalidParameter: input[N].reasoning ... Item reasoning is not supported for model`。 + 但你看到的**不是**报错：第一次后端调用成功，Agent 只跑了一条命令， + 这一轮就以 `status=completed` 结束，留下一个半成品工作区和一段像模像样的总结。 + 那个 400 只会以 `codex_backend_api_error` 警告的形式出现在日志里，**不会**抛给调用方。 + 请使用 `deepseek-v4-flash-260425`（已端到端验证）；当一个 codex 轮次结束得可疑地早时， + 先在日志里 grep 一下 `codex_backend_api_error`，再决定要不要相信那个回答。 +- **被桥接进 Codex 协议的聊天模型会“讲解”而不是“动手”。** Codex 收到 assistant 消息就结束这一轮， + 所以模型如果回一句*“我现在来写分析脚本”*，这一轮就会什么都没做地结束。 + 这正是 instruction 开头就写 *“Act, do not narrate”* 的原因。 + 在任何聊天后端上，都要预留一部分提示词预算来处理这件事。 +- **`apply_patch` 根本到不了后端，`request_user_input` 也没人能回答。** shim 只转发 + `function` 类型的工具，而 Codex 的文件编辑工具不是——后端实际看到的列表是 `exec_command`、 + `write_stdin`、`update_plan`、`request_user_input`、`view_image`，外加你自己的 ADK 工具。 + 但 Codex 自己的系统提示词仍然告诉模型去用 `apply_patch`；`request_user_input` 也照样被通告出去， + 尽管一次 ADK 调用根本没有可以回答它的交互通道。 + **runtime 现在会在每轮的 developer instructions 后面追加一段工具可用性说明**， + 把这两件事以及替代做法（用 `exec_command` 的 heredoc 写文件；自己拿主意而不是提问）讲清楚。 + 这个示例的 instruction 从前要手写这两句，现在不需要了。 + +一般性的教训是：在聊天后端上，这个 runtime 实际可用的工具面比 Codex 文档给人的印象要窄。 +这两个缺口 runtime 已经替你补上了，其余的仍然要靠你的 instruction。 + +## 约束 + +`runtime="codex"` 会直接拒绝 `Agent` 的一大片配置面，而不是默默忽略。与本例相关的有： + +- 被拒绝：`sub_agents`、`model=`（请用 `model_name=`）、`output_schema`、`planner`、 + `code_executor`、`include_contents="none"`、`enable_supervisor`， + 以及 `generate_content_config` 里除 `system_instruction` 之外的任何字段； +- 同样被拒绝：`CodexRuntimeConfig(sandbox="full_access", network_access=False)` + ——这个组合读起来像“没有网络”，实际却是全权限； +- 带告警丢弃：`knowledgebase`、`example_store`、`skills_mode` 等。 + +详见[支持矩阵](../../docs/content/docs/framework/agent/runtime.mdx#支持矩阵)。 diff --git a/examples/codex_data_analysis/analytics_tools.py b/examples/codex_data_analysis/analytics_tools.py new file mode 100644 index 000000000..18b9e9c3c --- /dev/null +++ b/examples/codex_data_analysis/analytics_tools.py @@ -0,0 +1,268 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The two ADK tools that bracket the sandbox: one imports data, one exports it. + +Both demonstrate the rule that matters most when combining ADK tools with +``runtime="codex"``: + + **The workspace is the data plane; tool arguments and results are the + control plane.** + +A tool result does not go into a file — the runtime's shim executes the tool +and feeds its JSON result back to the model *as text in the prompt*. So a tool +that returns 40 CSV rows pays for them in tokens on that request and on every +later request of the turn, and a tool that returns 40 000 rows breaks the turn. +Instead, :func:`fetch_sales_extract` writes the data into the Codex workspace +and returns a *receipt* — a path and a row count. The model then reads the file +with its own sandboxed code, where volume costs nothing. + +:func:`publish_report` is the same rule in reverse. Codex runs under +``sandbox="workspace_write"`` with ``network_access=False``, so it can write +only inside its workspace: it cannot email, upload, or copy a file anywhere +else. This tool is the single audited gate through which a finished artifact +leaves — which is why it validates the paths the model hands it rather than +trusting them. + +Both tools find the directory to write into with +:func:`veadk.runtime.codex.current_workspace`, which reports the workspace of +the turn that is calling the tool. Nothing here pins ``workspace_root``, so +each session gets its own directory and these tools follow whichever one they +are called from — the arrangement a multi-tenant server needs, and the one this +example therefore demonstrates. + +Requires: nothing beyond the standard library. The "internal system" is the +CSV under ``data/``. +""" + +from __future__ import annotations + +import csv +import hashlib +import re +import shutil +from datetime import datetime +from pathlib import Path + +from veadk.runtime.codex import current_workspace + +_HERE = Path(__file__).resolve().parent + +OUTBOX = _HERE / "outbox" +"""Where published artifacts land. Deliberately *outside* the workspace: the +sandbox cannot write here, so every file in it went through ``publish_report``.""" + +_WAREHOUSE = _HERE / "data" +"""Stands in for an internal reporting system (a warehouse, an ERP export).""" + +_QUARTER_RE = re.compile(r"^\d{4}q[1-4]$") + +_MAX_PUBLISH_BYTES = 2_000_000 + +_ALLOWED_SUFFIXES = {".md", ".svg"} + +_LAST_WORKSPACE: Path | None = None +"""The workspace the most recent tool call ran in — a *demo* affordance. + +``main.py`` prints the directory tree once the run is over, and this example +deliberately does not pin ``workspace_root``, so nothing outside a tool call +knows the path. A single-process, single-session script can remember it like +this; a server serving several sessions at once cannot, and does not need to — +its tools already receive the right directory on every call. +""" + + +def _workspace() -> Path | None: + """The workspace of the Codex turn calling this tool, or ``None``. + + Returns: + Path | None: Codex's working directory for this turn, or ``None`` when + the tool is running outside a codex turn (another runtime, an + ``AgentTool``, a unit test). + """ + global _LAST_WORKSPACE + workspace = current_workspace() + if workspace is None: + return None + _LAST_WORKSPACE = Path(workspace) + return _LAST_WORKSPACE + + +def last_seen_workspace() -> Path | None: + """The workspace observed by the last tool call. See :data:`_LAST_WORKSPACE`.""" + return _LAST_WORKSPACE + + +def _no_workspace_error() -> dict: + """The result to return when there is no workspace to write into. + + :func:`~veadk.runtime.codex.current_workspace` returns ``None`` rather than + raising when no codex turn is on the stack, and the tool answers in kind: + an error result the model can read beats an exception it cannot. + """ + return { + "status": "error", + "message": ( + "no sandbox working directory on this call, so nothing was " + "written; this tool only works inside a codex turn." + ), + } + + +def _resolve_in_workspace(workspace: Path, candidate: str) -> Path: + """Resolve a model-supplied path, refusing anything outside the workspace. + + The argument comes from the model, so it is untrusted input: ``..`` + segments, absolute paths and symlinks pointing out of the workspace are all + rejected here rather than trusted. The workspace arrives as an argument + because it is a property of the *call* — this turn's directory — not a + constant of the module. + + Args: + workspace (Path): This turn's workspace, from :func:`_workspace`. + candidate (str): Path as the model wrote it, relative to the workspace. + + Returns: + Path: The resolved, in-workspace path. + + Raises: + ValueError: If the path escapes the workspace. + """ + root = workspace.resolve() + resolved = (root / candidate).resolve() + if resolved != root and root not in resolved.parents: + raise ValueError(f"path escapes the workspace: {candidate!r}") + return resolved + + +def fetch_sales_extract(quarter: str) -> dict: + """Export one quarter of raw order data from the internal sales warehouse. + + The rows are written into a file in your working directory. This tool + returns only a receipt — it never returns the data itself, so read the CSV + at the returned path with your own code. + + Args: + quarter (str): Fiscal quarter to export, e.g. ``"2025Q3"``. + + Returns: + dict: On success, ``status``, ``path`` (relative to your working + directory), ``rows``, ``columns`` and ``bytes``. On failure, + ``status="error"`` and a ``message`` saying what to try instead. + """ + workspace = _workspace() + if workspace is None: + return _no_workspace_error() + + normalized = quarter.strip().lower().replace("-", "").replace(" ", "") + if not _QUARTER_RE.match(normalized): + return { + "status": "error", + "message": f"{quarter!r} is not a quarter; expected e.g. '2025Q3'.", + } + + source = _WAREHOUSE / f"sales_{normalized}.csv" + if not source.is_file(): + available = sorted( + path.stem.removeprefix("sales_").upper() + for path in _WAREHOUSE.glob("sales_*.csv") + ) + return { + "status": "error", + "message": f"no extract for {quarter!r}; available: {available}", + } + + destination = workspace / "data" / source.name + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + + with source.open(newline="", encoding="utf-8") as handle: + reader = csv.reader(handle) + header = next(reader, []) + rows = sum(1 for _ in reader) + + return { + "status": "ok", + "path": str(destination.relative_to(workspace)), + "rows": rows, + "columns": header, + "bytes": destination.stat().st_size, + "note": "raw warehouse export, exactly as stored upstream", + } + + +def publish_report(report_path: str, chart_path: str) -> dict: + """Publish a finished report and its chart to the reporting outbox. + + This is the only way a file leaves your sandbox: you have no network + access and cannot write outside your working directory. Call it once the + report is complete. + + Args: + report_path (str): The Markdown report, relative to your working + directory (must end in ``.md``). + chart_path (str): The SVG chart, relative to your working directory + (must end in ``.svg``). + + Returns: + dict: On success, ``status``, ``published`` (one entry per file with + its destination, size and content digest) and ``published_at``. On + failure, ``status="error"`` and a ``message`` saying what to fix. + """ + workspace = _workspace() + if workspace is None: + return _no_workspace_error() + + try: + sources = [ + _resolve_in_workspace(workspace, p) for p in (report_path, chart_path) + ] + except ValueError as error: + return {"status": "error", "message": str(error)} + + for path, original in zip(sources, (report_path, chart_path)): + if path.suffix.lower() not in _ALLOWED_SUFFIXES: + return { + "status": "error", + "message": f"{original!r}: only .md and .svg files are published", + } + if not path.is_file(): + return {"status": "error", "message": f"{original!r}: no such file"} + size = path.stat().st_size + if size == 0: + return {"status": "error", "message": f"{original!r}: file is empty"} + if size > _MAX_PUBLISH_BYTES: + return { + "status": "error", + "message": f"{original!r}: {size} bytes exceeds the publish limit", + } + + published_at = datetime.now().strftime("%Y%m%d-%H%M%S") + destination_dir = OUTBOX / published_at + destination_dir.mkdir(parents=True, exist_ok=True) + + published = [] + for path in sources: + destination = destination_dir / path.name + shutil.copyfile(path, destination) + payload = destination.read_bytes() + published.append( + { + "file": str(destination.relative_to(OUTBOX)), + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest()[:12], + } + ) + + return {"status": "ok", "published": published, "published_at": published_at} diff --git a/examples/codex_data_analysis/data/sales_2025q3.csv b/examples/codex_data_analysis/data/sales_2025q3.csv new file mode 100644 index 000000000..ed35d44b4 --- /dev/null +++ b/examples/codex_data_analysis/data/sales_2025q3.csv @@ -0,0 +1,2401 @@ +order_id,order_date,region,channel,sku,units,revenue_cny +SO-100001,2025-07-01,South,direct,VE-1000,8,2628.00 +SO-100002,2025-07-01,East,partner,VE-1000,6,2080.50 +SO-100003,2025-07-01,East,direct,VE-1000,4,1460.00 +SO-100004,2025-07-01,North,online,VE-3000,10,19900.00 +SO-100005,2025-07-01,East,direct,VE-2000,6,2700.00 +SO-100006,2025-07-01,East,partner,VE-1000,4,1460.00 +SO-100007,2025-07-01,East,online,VE-1000,7,2555.00 +SO-100008,2025-07-01,North,online,VE-1000,10,3650.00 +SO-100009,2025-07-01,South,online,VE-2000,1,405.00 +SO-100010,2025-07-01,North,partner,VE-2000,3,1350.00 +SO-100011,2025-07-01,South,partner,VE-3000,1,1990.00 +SO-100012,2025-07-01,East,partner,VE-2000,5,2250.00 +SO-100013,2025-07-01,South,online,VE-1000,17,6205.00 +SO-100014,2025-07-01,South,partner,VE-2000,11,4702.50 +SO-100015,2025-07-01,North,direct,VE-1000,17,6205.00 +SO-100016,2025-07-01,North,online,VE-2000,4,1800.00 +SO-100017,2025-07-01,South,online,VE-1000,11,3613.50 +SO-100018,2025-07-01,West,online,VE-2000,8,3600.00 +SO-100019,2025-07-01,North,partner,VE-2000,17,7267.50 +SO-100020,2025-07-01,South,online,VE-1000,9,2956.50 +SO-100021,2025-07-01,North,online,VE-2000,5,2250.00 +SO-100022,2025-07-01,South,online,VE-2000,9,3847.50 +SO-100023,2025-07-02,East,direct,VE-2000,7,3150.00 +SO-100024,2025-07-02,North,partner,VE-3000,7,13930.00 +SO-100025,2025-07-02,North,online,VE-2000,5,2250.00 +SO-100026,2025-07-02,South,online,VE-1000,9,2956.50 +SO-100027,2025-07-02,North,online,VE-1000,18,6570.00 +SO-100028,2025-07-02,West,online,VE-1000,7,2427.25 +SO-100029,2025-07-02,South,partner,VE-1000,14,5110.00 +SO-100030,2025-07-02,North,partner,VE-1000,7,2555.00 +SO-100031,2025-07-02,West,online,VE-1000,2,730.00 +SO-100032,2025-07-02,West,direct,VE-2000,3,1350.00 +SO-100033,2025-07-02,East,online,VE-3000,12,23880.00 +SO-100034,2025-07-02,West,partner,VE-3000,11,21890.00 +SO-100035,2025-07-02,North,online,VE-2000,20,8100.00 +SO-100036,2025-07-02,North,direct,VE-1000,5,1825.00 +SO-100037,2025-07-02,West,online,VE-3000,10,17910.00 +SO-100038,2025-07-02,West,online,VE-2000,1,427.50 +SO-100039,2025-07-02,East,direct,VE-1000,10,3467.50 +SO-100040,2025-07-02,South,direct,VE-1000,1,365.00 +SO-100041,2025-07-02,North,partner,VE-2000,4,1800.00 +SO-100042,2025-07-02,East,partner,VE-3000,16,31840.00 +SO-100043,2025-07-02,East,direct,VE-1000,13,4507.75 +SO-100044,2025-07-02,West,direct,VE-1000,9,3285.00 +SO-100045,2025-07-02,East,direct,VE-3000,7,12537.00 +SO-100046,2025-07-02,South,partner,VE-1000,12,3942.00 +SO-100047,2025-07-02,East,online,VE-2000,7,3150.00 +SO-100048,2025-07-02,South,online,VE-2000,5,2250.00 +SO-100049,2025-07-02,South,direct,VE-1000,8,2920.00 +SO-100050,2025-07-02,South,direct,VE-3000,6,11940.00 +SO-100051,2025-07-03,East,direct,VE-1000,4,1460.00 +SO-100052,2025-07-03,North,direct,VE-1000,11,3613.50 +SO-100053,2025-07-03,North,partner,VE-2000,3,1350.00 +SO-100054,2025-07-03,West,online,VE-1000,9,3120.75 +SO-100055,2025-07-03,South,partner,VE-2000,9,3645.00 +SO-100056,2025-07-03,West,online,VE-2000,12,5400.00 +SO-100057,2025-07-03,West,partner,VE-1000,7,2427.25 +SO-100058,2025-07-03,South,partner,VE-1000,1,346.75 +SO-100059,2025-07-03,West,direct,VE-1000,4,1460.00 +SO-100060,2025-07-03,South,direct,VE-1000,8,2920.00 +SO-100061,2025-07-03,North,direct,VE-2000,8,3600.00 +SO-100062,2025-07-03,North,online,VE-2000,19,8122.50 +SO-100063,2025-07-03,North,partner,VE-1000,22,7628.50 +SO-100064,2025-07-03,West,partner,VE-2000,11,4950.00 +SO-100065,2025-07-03,North,online,VE-1000,8,2920.00 +SO-100066,2025-07-03,North,online,VE-3000,5,9452.50 +SO-100067,2025-07-03,West,direct,VE-1000,3,1095.00 +SO-100068,2025-07-03,East,direct,VE-1000,3,1095.00 +SO-100069,2025-07-03,South,partner,VE-1000,11,3613.50 +SO-100070,2025-07-03,North,direct,VE-3000,2,3980.00 +SO-100071,2025-07-03,East,partner,VE-2000,3,1350.00 +SO-100072,2025-07-03,West,online,VE-2000,7,2835.00 +SO-100073,2025-07-03,South,direct,VE-2000,3,1350.00 +SO-100074,2025-07-03,South,partner,VE-2000,5,2250.00 +SO-100075,2025-07-03,West,online,VE-1000,15,5475.00 +SO-100076,2025-07-04,East,direct,VE-1000,8,2774.00 +SO-100077,2025-07-04,East,partner,VE-2000,13,5265.00 +SO-100078,2025-07-04,East,partner,VE-1000,15,4927.50 +SO-100079,2025-07-04,South,online,VE-1000,1,365.00 +SO-100080,2025-07-04,East,online,VE-1000,12,3942.00 +SO-100081,2025-07-04,South,direct,VE-1000,23,7975.25 +SO-100082,2025-07-04,North,partner,VE-2000,1,450.00 +SO-100083,2025-07-04,West,online,VE-1000,8,2920.00 +SO-100084,2025-07-04,South,partner,VE-2000,17,6885.00 +SO-100085,2025-07-04,South,direct,VE-2000,15,6412.50 +SO-100086,2025-07-04,West,partner,VE-1000,12,4380.00 +SO-100087,2025-07-04,East,online,VE-2000,6,2565.00 +SO-100088,2025-07-04,South,online,VE-2000,7,2835.00 +SO-100089,2025-07-04,North,direct,VE-1000,5,1825.00 +SO-100090,2025-07-04,West,partner,VE-2000,1,427.50 +SO-100091,2025-07-04,South,online,VE-1000,1,365.00 +SO-100092,2025-07-04,North,direct,VE-1000,11,3814.25 +SO-100093,2025-07-04,North,partner,VE-2000,7,3150.00 +SO-100094,2025-07-04,South,online,VE-1000,9,3285.00 +SO-100095,2025-07-04,East,direct,VE-2000,6,2700.00 +SO-100096,2025-07-04,West,online,VE-1000,10,3650.00 +SO-100097,2025-07-04,West,partner,VE-1000,6,2190.00 +SO-100098,2025-07-04,West,direct,VE-1000,10,3650.00 +SO-100099,2025-07-04,East,direct,VE-3000,4,7960.00 +SO-100100,2025-07-04,South,direct,VE-2000,3,1282.50 +SO-100101,2025-07-04,South,partner,VE-2000,4,1800.00 +SO-100102,2025-07-05,South,direct,VE-2000,15,6412.50 +SO-100103,2025-07-05,South,direct,VE-2000,1,405.00 +SO-100104,2025-07-05,North,partner,VE-1000,17,6205.00 +SO-100105,2025-07-05,East,partner,VE-1000,17,5584.50 +SO-100106,2025-07-05,South,direct,VE-1000,3,1095.00 +SO-100107,2025-07-05,East,partner,VE-2000,6,2565.00 +SO-100108,2025-07-05,North,online,VE-3000,4,7960.00 +SO-100109,2025-07-05,North,direct,VE-1000,7,2427.25 +SO-100110,2025-07-05,West,partner,VE-3000,9,17014.50 +SO-100111,2025-07-05,East,direct,VE-1000,5,1825.00 +SO-100112,2025-07-05,West,online,VE-2000,23,9315.00 +SO-100113,2025-07-05,North,partner,VE-1000,4,1460.00 +SO-100114,2025-07-05,West,partner,VE-3000,4,7960.00 +SO-100115,2025-07-05,East,direct,VE-1000,1,365.00 +SO-100116,2025-07-05,East,partner,VE-2000,1,450.00 +SO-100117,2025-07-05,North,partner,VE-1000,8,2774.00 +SO-100118,2025-07-05,West,partner,VE-2000,7,3150.00 +SO-100119,2025-07-05,North,partner,VE-1000,1,365.00 +SO-100120,2025-07-05,West,online,VE-1000,1,365.00 +SO-100121,2025-07-05,South,direct,VE-1000,9,3285.00 +SO-100122,2025-07-05,South,direct,VE-1000,15,5201.25 +SO-100123,2025-07-05,West,direct,VE-3000,15,26865.00 +SO-100124,2025-07-05,East,partner,VE-1000,8,2920.00 +SO-100125,2025-07-05,North,online,VE-1000,22,7227.00 +SO-100126,2025-07-05,North,partner,VE-1000,7,2555.00 +SO-100127,2025-07-05,South,direct,VE-2000,13,5850.00 +SO-100128,2025-07-05,East,online,VE-1000,9,3285.00 +SO-100129,2025-07-05,North,partner,VE-1000,5,1825.00 +SO-100130,2025-07-05,North,direct,VE-1000,10,3650.00 +SO-100131,2025-07-05,East,online,VE-1000,12,4380.00 +SO-100132,2025-07-06,South,direct,VE-2000,11,4702.50 +SO-100133,2025-07-06,North,online,VE-1000,9,3120.75 +SO-100134,2025-07-06,West,partner,VE-1000,14,4599.00 +SO-100135,2025-07-06,South,online,VE-1000,1,328.50 +SO-100136,2025-07-06,East,partner,VE-3000,12,23880.00 +SO-100137,2025-07-06,South,online,VE-2000,9,4050.00 +SO-100138,2025-07-06,South,direct,VE-1000,10,3650.00 +SO-100139,2025-07-06,East,partner,VE-2000,1,450.00 +SO-100140,2025-07-06,East,direct,VE-1000,5,1825.00 +SO-100141,2025-07-06,West,direct,VE-1000,4,1460.00 +SO-100142,2025-07-06,East,partner,VE-1000,6,2190.00 +SO-100143,2025-07-06,East,partner,VE-1000,1,346.75 +SO-100144,2025-07-06,South,direct,VE-1000,16,5256.00 +SO-100145,2025-07-06,West,direct,VE-1000,9,3285.00 +SO-100146,2025-07-06,East,online,VE-1000,11,3814.25 +SO-100147,2025-07-06,South,direct,VE-1000,14,5110.00 +SO-100148,2025-07-06,South,partner,VE-1000,6,1971.00 +SO-100149,2025-07-06,South,partner,VE-1000,5,1733.75 +SO-100150,2025-07-06,West,partner,VE-3000,1,1990.00 +SO-100151,2025-07-06,North,partner,VE-3000,6,10746.00 +SO-100152,2025-07-06,East,direct,VE-2000,16,6480.00 +SO-100153,2025-07-06,East,online,VE-2000,5,2025.00 +SO-100154,2025-07-06,South,partner,VE-1000,9,2956.50 +SO-100155,2025-07-06,East,partner,VE-2000,3,1215.00 +SO-100156,2025-07-07,North,direct,VE-1000,5,1825.00 +SO-100157,2025-07-07,West,online,VE-2000,12,5400.00 +SO-100158,2025-07-07,East,partner,VE-3000,11,21890.00 +SO-100159,2025-07-07,South,direct,VE-1000,18,6570.00 +SO-100160,2025-07-07,East,direct,VE-3000,16,31840.00 +SO-100161,2025-07-07,South,partner,VE-1000,5,1825.00 +SO-100162,2025-07-07,West,partner,VE-1000,1,328.50 +SO-100163,2025-07-07,South,online,VE-1000,9,2956.50 +SO-100164,2025-07-07,South,direct,VE-2000,15,6750.00 +SO-100165,2025-07-07,South,partner,VE-1000,1,365.00 +SO-100166,2025-07-07,South,partner,VE-1000,6,1971.00 +SO-100167,2025-07-07,South,direct,VE-1000,11,3613.50 +SO-100168,2025-07-07,South,direct,VE-3000,5,9950.00 +SO-100169,2025-07-07,North,partner,VE-1000,20,7300.00 +SO-100170,2025-07-07,South,online,VE-3000,5,9950.00 +SO-100171,2025-07-07,East,direct,VE-2000,9,3645.00 +SO-100172,2025-07-07,North,direct,VE-1000,14,5110.00 +SO-100173,2025-07-07,North,direct,VE-2000,17,6885.00 +SO-100174,2025-07-07,South,online,VE-1000,2,730.00 +SO-100175,2025-07-07,West,direct,VE-2000,16,7200.00 +SO-100176,2025-07-07,East,direct,VE-1000,1,365.00 +SO-100177,2025-07-07,West,direct,VE-2000,1,450.00 +SO-100178,2025-07-07,West,direct,VE-3000,12,22686.00 +SO-100179,2025-07-07,East,direct,VE-2000,4,1800.00 +SO-100180,2025-07-07,North,direct,VE-2000,3,1215.00 +SO-100181,2025-07-07,East,online,VE-3000,9,17910.00 +SO-100182,2025-07-07,North,direct,VE-2000,9,4050.00 +SO-100183,2025-07-07,South,direct,VE-2000,8,3240.00 +SO-100184,2025-07-07,South,partner,VE-1000,10,3467.50 +SO-100185,2025-07-07,North,online,VE-1000,12,4380.00 +SO-100186,2025-07-08,West,partner,VE-1000,20,7300.00 +SO-100187,2025-07-08,South,online,VE-1000,2,730.00 +SO-100188,2025-07-08,East,online,VE-1000,20,6570.00 +SO-100189,2025-07-08,East,partner,VE-2000,1,427.50 +SO-100190,2025-07-08,South,online,VE-3000,11,19701.00 +SO-100191,2025-07-08,North,partner,VE-1000,18,6570.00 +SO-100192,2025-07-08,West,direct,VE-3000,5,9950.00 +SO-100193,2025-07-08,South,online,VE-3000,1,1890.50 +SO-100194,2025-07-08,North,online,VE-2000,17,7267.50 +SO-100195,2025-07-08,North,online,VE-3000,13,24576.50 +SO-100196,2025-07-08,South,partner,VE-2000,4,1800.00 +SO-100197,2025-07-08,North,partner,VE-2000,14,5670.00 +SO-100198,2025-07-08,West,online,VE-3000,14,26467.00 +SO-100199,2025-07-08,North,direct,VE-3000,5,9452.50 +SO-100200,2025-07-08,North,direct,VE-1000,4,1460.00 +SO-100201,2025-07-08,East,partner,VE-1000,9,3285.00 +SO-100202,2025-07-08,West,partner,VE-3000,3,5970.00 +SO-100203,2025-07-08,North,direct,VE-1000,19,6935.00 +SO-100204,2025-07-08,East,online,VE-2000,1,405.00 +SO-100205,2025-07-08,South,online,VE-2000,12,4860.00 +SO-100206,2025-07-08,West,partner,VE-1000,1,365.00 +SO-100207,2025-07-08,South,partner,VE-3000,6,10746.00 +SO-100208,2025-07-08,West,direct,VE-2000,1,450.00 +SO-100209,2025-07-08,South,partner,VE-1000,7,2555.00 +SO-100210,2025-07-08,East,direct,VE-2000,1,450.00 +SO-100211,2025-07-08,East,online,VE-1000,10,"3,650.00" +SO-100212,2025-07-08,South,direct,VE-1000,4,1460.00 +SO-100213,2025-07-08,South,online,VE-1000,11,3814.25 +SO-100214,2025-07-08,South,partner,VE-2000,17,7650.00 +SO-100215,2025-07-08,North,direct,VE-1000,8,2628.00 +SO-100216,2025-07-08,North,partner,VE-2000,1,405.00 +SO-100217,2025-07-08,South,online,VE-1000,2,730.00 +SO-100218,2025-07-08,South,partner,VE-3000,11,19701.00 +SO-100219,2025-07-08,East,online,VE-1000,18,6570.00 +SO-100220,2025-07-08,South,online,VE-2000,11,4455.00 +SO-100221,2025-07-09,South,online,VE-1000,7,2555.00 +SO-100222,2025-07-09,North,partner,VE-1000,11,3814.25 +SO-100223,2025-07-09,South,direct,VE-3000,7,13930.00 +SO-100224,2025-07-09,South,online,VE-2000,13,5557.50 +SO-100225,2025-07-09,North,partner,VE-1000,5,1733.75 +SO-100226,2025-07-09,East,direct,VE-1000,15,5475.00 +SO-100227,2025-07-09,South,direct,VE-2000,6,"2,430.00" +SO-100228,2025-07-09,North,partner,VE-1000,4,1387.00 +SO-100229,2025-07-09,West,online,VE-2000,1,427.50 +SO-100230,2025-07-09,North,direct,VE-2000,4,1710.00 +SO-100231,2025-07-09,East,online,VE-2000,4,1800.00 +SO-100232,2025-07-09,East,online,VE-2000,10,4050.00 +SO-100233,2025-07-09,West,online,VE-2000,19,8122.50 +SO-100234,2025-07-09,North,partner,VE-1000,8,2920.00 +SO-100235,2025-07-09,East,online,VE-1000,17,6205.00 +SO-100236,2025-07-09,East,direct,VE-2000,16,7200.00 +SO-100237,2025-07-09,North,direct,VE-2000,7,"3,150.00" +SO-100238,2025-07-09,South,partner,VE-2000,7,2992.50 +SO-100239,2025-07-09,South,direct,VE-1000,1,365.00 +SO-100240,2025-07-09,East,direct,VE-3000,5,9950.00 +SO-100241,07/09/2025,South,online,VE-2000,1,405.00 +SO-100242,2025-07-09,South,online,VE-2000,4,1710.00 +SO-100243,2025-07-10,South,direct,VE-2000,9,4050.00 +SO-100244,2025-07-10,South,partner,VE-2000,9,3847.50 +SO-100245,2025-07-10,East,online,VE-1000,8,2628.00 +SO-100246,2025-07-10,South,online,VE-2000,8,3600.00 +SO-100247,2025-07-10,East,direct,VE-1000,5,1825.00 +SO-100248,2025-07-10,East,online,VE-2000,17,6885.00 +SO-100249,2025-07-10,East,online,VE-2000,2,900.00 +SO-100250,2025-07-10,North,partner,VE-1000,4,1460.00 +SO-100251,2025-07-10,North,online,VE-3000,8,15920.00 +SO-100252,2025-07-10,North,direct,VE-1000,16,5840.00 +SO-100253,2025-07-10,South,direct,VE-2000,18,8100.00 +SO-100254,2025-07-10,North,partner,VE-2000,9,4050.00 +SO-100255,2025-07-10,West,partner,VE-1000,4,1460.00 +SO-100256,2025-07-10,East,partner,VE-2000,3,1350.00 +SO-100257,2025-07-10,North,online,VE-1000,9,3285.00 +SO-100258,2025-07-10,East,online,VE-1000,7,2555.00 +SO-100259,2025-07-10,North,direct,VE-1000,10,3467.50 +SO-100260,2025-07-10,South,direct,VE-1000,5,1642.50 +SO-100261,2025-07-10,North,direct,VE-1000,21,7665.00 +SO-100262,2025-07-10,East,partner,VE-1000,8,2920.00 +SO-100263,2025-07-10,East,partner,VE-1000,6,2190.00 +SO-100264,2025-07-10,West,online,VE-2000,9,4050.00 +SO-100265,2025-07-10,West,direct,VE-1000,8,"2,920.00" +SO-100266,2025-07-10,East,online,VE-1000,15,5475.00 +SO-100267,2025-07-10,South,online,VE-1000,11,4015.00 +SO-100268,2025-07-10,North,partner,VE-2000,10,4050.00 +SO-100269,2025-07-10,North,partner,VE-2000,14,5985.00 +SO-100270,2025-07-10,West,online,VE-2000,1,450.00 +SO-100271,2025-07-10,East,partner,VE-1000,18,6570.00 +SO-100272,2025-07-10,West,online,VE-2000,20,9000.00 +SO-100273,2025-07-10,North,partner,VE-1000,2,657.00 +SO-100274,2025-07-10,West,partner,VE-1000,15,5475.00 +SO-100275,2025-07-10,North,direct,VE-3000,5,8955.00 +SO-100276,2025-07-11,South,partner,VE-1000,1,365.00 +SO-100277,2025-07-11,West,partner,VE-3000,11,21890.00 +SO-100278,2025-07-11,North,online,VE-1000,10,3650.00 +SO-100279,2025-07-11,West,online,VE-1000,1,365.00 +SO-100280,2025-07-11,North,online,VE-1000,10,3650.00 +SO-100281,2025-07-11,North,online,VE-2000,1,405.00 +SO-100282,2025-07-11,West,direct,VE-3000,12,23880.00 +SO-100283,2025-07-11,South,partner,VE-1000,13,4270.50 +SO-100284,2025-07-11,West,partner,VE-2000,14,6300.00 +SO-100285,2025-07-11,East,direct,VE-2000,3,1350.00 +SO-100286,2025-07-11,West,partner,VE-2000,4,1800.00 +SO-100287,2025-07-11,East,direct,VE-2000,2,810.00 +SO-100288,2025-07-11,South,direct,VE-1000,11,4015.00 +SO-100289,2025-07-11,South,partner,VE-2000,1,450.00 +SO-100290,2025-07-11,West,online,VE-3000,8,"15,124.00" +SO-100291,2025-07-11,North,partner,VE-1000,9,2956.50 +SO-100292,2025-07-11,South,partner,VE-1000,1,365.00 +SO-100293,2025-07-11,North,direct,VE-1000,7,2427.25 +SO-100294,2025-07-11,West,direct,VE-2000,7,2835.00 +SO-100295,2025-07-11,West,online,VE-2000,11,4702.50 +SO-100296,2025-07-11,North,direct,VE-3000,14,26467.00 +SO-100297,2025-07-11,West,online,VE-1000,1,365.00 +SO-100298,2025-07-11,East,direct,VE-2000,12,5400.00 +SO-100299,2025-07-11,West,partner,VE-1000,7,2427.25 +SO-100300,2025-07-11,North,partner,VE-2000,2,810.00 +SO-100301,2025-07-11,West,online,VE-1000,1,328.50 +SO-100302,2025-07-11,North,partner,VE-2000,10,4500.00 +SO-100303,07/11/2025,South,partner,VE-2000,16,6480.00 +SO-100304,2025-07-11,North,direct,VE-2000,5,2250.00 +SO-100305,2025-07-11,South,direct,VE-2000,1,427.50 +SO-100306,2025-07-11,South,partner,VE-3000,9,16119.00 +SO-100307,2025-07-11,North,partner,VE-1000,7,2299.50 +SO-100308,2025-07-11,East,online,VE-1000,1,365.00 +SO-100309,2025-07-11,East,online,VE-1000,11,3613.50 +SO-100310,2025-07-11,East,partner,VE-2000,10,4500.00 +SO-100311,2025-07-12,North,partner,VE-1000,5,1733.75 +SO-100312,2025-07-12,East,online,VE-2000,5,2250.00 +SO-100313,2025-07-12,North,partner,VE-2000,13,5557.50 +SO-100314,2025-07-12,West,online,VE-2000,14,6300.00 +SO-100315,2025-07-12,North,direct,VE-1000,13,4507.75 +SO-100316,2025-07-12,North,direct,VE-1000,9,3120.75 +SO-100317,2025-07-12,South,partner,VE-3000,10,19900.00 +SO-100318,2025-07-12,South,online,VE-2000,21,9450.00 +SO-100319,2025-07-12,North,direct,VE-3000,1,1990.00 +SO-100320,2025-07-12,East,direct,VE-1000,25,9125.00 +SO-100321,2025-07-12,West,direct,VE-1000,7,2427.25 +SO-100322,2025-07-12,East,partner,VE-1000,10,3467.50 +SO-100323,2025-07-12,West,partner,VE-1000,11,3613.50 +SO-100324,2025-07-12,West,direct,VE-2000,5,2137.50 +SO-100325,2025-07-12,South,online,VE-3000,16,28656.00 +SO-100326,2025-07-12,North,online,VE-2000,1,405.00 +SO-100327,2025-07-12,South,direct,VE-1000,8,2920.00 +SO-100328,2025-07-12,West,online,VE-2000,15,6412.50 +SO-100329,2025-07-12,South,online,VE-2000,11,4455.00 +SO-100330,2025-07-12,North,online,VE-3000,2,3980.00 +SO-100331,2025-07-12,South,online,VE-1000,6,2190.00 +SO-100332,2025-07-12,South,direct,VE-2000,20,9000.00 +SO-100333,2025-07-12,West,direct,VE-3000,5,9950.00 +SO-100334,2025-07-12,South,direct,VE-1000,7,2555.00 +SO-100335,2025-07-12,North,online,VE-1000,16,5548.00 +SO-100336,2025-07-12,South,online,VE-2000,6,2700.00 +SO-100337,2025-07-13,South,online,VE-3000,14,27860.00 +SO-100338,2025-07-13,south,online,VE-1000,1,365.00 +SO-100339,2025-07-13,East,partner,VE-1000,10,"3,650.00" +SO-100340,2025-07-13,North,online,VE-1000,7,2427.25 +SO-100341,2025-07-13,East,online,VE-1000,6,2190.00 +SO-100342,2025-07-13,West,online,VE-2000,8,3240.00 +SO-100343,2025-07-13,South,direct,VE-1000,12,4380.00 +SO-100344,2025-07-13,North,online,VE-1000,11,3613.50 +SO-100345,2025-07-13,West,direct,VE-3000,17,33830.00 +SO-100346,2025-07-13,North,online,VE-3000,10,19900.00 +SO-100347,2025-07-13,North,online,VE-2000,11,4950.00 +SO-100348,2025-07-13,South,partner,VE-1000,14,5110.00 +SO-100349,2025-07-13,North,partner,VE-1000,19,6241.50 +SO-100350,2025-07-13,North,direct,VE-2000,3,1282.50 +SO-100351,2025-07-13,East,direct,VE-2000,12,5400.00 +SO-100352,2025-07-13,North,partner,VE-3000,6,10746.00 +SO-100353,2025-07-13,North,direct,VE-1000,1,328.50 +SO-100354,2025-07-13,North,online,VE-3000,8,15920.00 +SO-100355,2025-07-13,West,online,VE-1000,1,365.00 +SO-100356,2025-07-13,North,online,VE-1000,15,4927.50 +SO-100357,2025-07-13,West,partner,VE-1000,1,365.00 +SO-100358,2025-07-13,South,online,VE-1000,2,730.00 +SO-100359,2025-07-14,East,direct,VE-3000,10,19900.00 +SO-100360,2025-07-14,North,direct,VE-3000,5,9452.50 +SO-100361,2025-07-14,North,direct,VE-2000,14,5985.00 +SO-100362,2025-07-14,South,online,VE-1000,8,2920.00 +SO-100363,2025-07-14,South,online,VE-1000,7,2427.25 +SO-100364,2025-07-14,North,direct,VE-2000,5,2025.00 +SO-100365,2025-07-14,North,online,VE-2000,10,4275.00 +SO-100366,2025-07-14,South,partner,VE-2000,6,2565.00 +SO-100367,2025-07-14,East,direct,VE-2000,12,5400.00 +SO-100368,2025-07-14,West,direct,VE-2000,4,1800.00 +SO-100369,2025-07-14,East,direct,VE-1000,4,1314.00 +SO-100370,2025-07-14,North,partner,VE-1000,1,328.50 +SO-100371,2025-07-14,South,direct,VE-1000,4,1460.00 +SO-100372,2025-07-14,South,online,VE-1000,3,1095.00 +SO-100373,2025-07-14,West,direct,VE-2000,9,3645.00 +SO-100374,2025-07-14,West,partner,VE-1000,4,1460.00 +SO-100375,2025-07-14,North,direct,VE-1000,8,2920.00 +SO-100376,2025-07-14,North,direct,VE-2000,10,4500.00 +SO-100377,2025-07-14,West,direct,VE-2000,10,4050.00 +SO-100378,2025-07-14,North,online,VE-3000,9,16119.00 +SO-100379,2025-07-14,North,partner,VE-1000,7,2555.00 +SO-100380,2025-07-14,East,direct,VE-2000,10,4050.00 +SO-100381,2025-07-14,South,direct,VE-3000,6,11343.00 +SO-100382,2025-07-14,North,online,VE-1000,1,346.75 +SO-100383,2025-07-14,West,partner,VE-2000,6,2700.00 +SO-100384,2025-07-14,North,online,VE-1000,18,6241.50 +SO-100385,2025-07-14,East,online,VE-1000,13,4745.00 +SO-100386,2025-07-14,East,direct,VE-1000,4,1387.00 +SO-100387,2025-07-15,South,partner,VE-3000,5,9950.00 +SO-100388,2025-07-15,North,partner,VE-3000,9,16119.00 +SO-100389,2025-07-15,West,partner,VE-3000,9,16119.00 +SO-100390,2025-07-15,West,partner,VE-1000,3,1095.00 +SO-100391,2025-07-15,North,partner,VE-3000,9,16119.00 +SO-100392,2025-07-15,North,partner,VE-1000,7,2555.00 +SO-100393,2025-07-15,East,online,VE-3000,10,18905.00 +SO-100394,2025-07-15,West,direct,VE-1000,12,"4,380.00" +SO-100395,2025-07-15,West,direct,VE-3000,15,29850.00 +SO-100396,2025-07-15,North,direct,VE-3000,14,27860.00 +SO-100397,2025-07-15,South,partner,VE-1000,1,365.00 +SO-100398,2025-07-15,North,partner,VE-2000,8,3600.00 +SO-100399,2025-07-15,East,partner,VE-1000,8,2920.00 +SO-100400,2025-07-15,East,partner,VE-3000,3,5671.50 +SO-100401,2025-07-15,West,online,VE-1000,2,730.00 +SO-100402,2025-07-15,North,online,VE-2000,16,7200.00 +SO-100403,2025-07-15,North,direct,VE-2000,8,3600.00 +SO-100404,2025-07-15,South,direct,VE-3000,16,31840.00 +SO-100405,2025-07-15,West,partner,VE-1000,9,3285.00 +SO-100406,2025-07-15,West,direct,VE-1000,11,4015.00 +SO-100407,2025-07-15,West,partner,VE-2000,1,405.00 +SO-100408,2025-07-15,East,partner,VE-2000,7,2835.00 +SO-100409,2025-07-15,East,online,VE-1000,19,6588.25 +SO-100410,2025-07-15,East,direct,VE-1000,17,6205.00 +SO-100411,2025-07-15,North,online,VE-3000,2,3980.00 +SO-100412,2025-07-15,East,partner,VE-3000,5,8955.00 +SO-100413,2025-07-15,West,direct,VE-3000,13,25870.00 +SO-100414,2025-07-16,South,partner,VE-3000,19,34029.00 +SO-100415,2025-07-16,East,online,VE-2000,7,2992.50 +SO-100416,2025-07-16,South,direct,VE-1000,11,4015.00 +SO-100417,2025-07-16,East,partner,VE-1000,11,4015.00 +SO-100418,2025-07-16,South,direct,VE-1000,16,5840.00 +SO-100419,2025-07-16,North,online,VE-1000,17,6205.00 +SO-100420,2025-07-16,North,direct,VE-1000,12,4380.00 +SO-100421,2025-07-16,West,partner,VE-1000,11,4015.00 +SO-100422,2025-07-16,North,direct,VE-1000,8,2774.00 +SO-100423,2025-07-16,East,partner,VE-1000,7,2427.25 +SO-100424,2025-07-16,East,partner,VE-2000,5,2250.00 +SO-100425,2025-07-16,North,online,VE-1000,9,2956.50 +SO-100426,2025-07-16,North,partner,VE-1000,10,3650.00 +SO-100427,2025-07-16,North,partner,VE-3000,1,1791.00 +SO-100428,2025-07-16,East,direct,VE-2000,11,4950.00 +SO-100429,2025-07-16,North,direct,VE-1000,1,328.50 +SO-100430,2025-07-16,South,online,VE-1000,20,7300.00 +SO-100431,2025-07-16,South,direct,VE-2000,8,3600.00 +SO-100432,2025-07-16,North,online,VE-3000,4,7562.00 +SO-100433,2025-07-16,South,partner,VE-1000,8,2920.00 +SO-100434,2025-07-16,East,partner,VE-2000,11,4950.00 +SO-100435,2025-07-16,North,direct,VE-2000,16,7200.00 +SO-100436,2025-07-16,West,partner,VE-1000,17,6205.00 +SO-100437,2025-07-16,West,partner,VE-2000,5,2250.00 +SO-100438,2025-07-16,South,partner,VE-1000,13,4745.00 +SO-100439,2025-07-16,West,partner,VE-1000,12,4380.00 +SO-100440,2025-07-16,South,partner,VE-1000,13,4507.75 +SO-100441,2025-07-17,West,partner,VE-2000,1,405.00 +SO-100442,2025-07-17,East,online,VE-1000,20,7300.00 +SO-100443,2025-07-17,South,direct,VE-1000,6,2190.00 +SO-100444,2025-07-17,North,online,VE-1000,19,6241.50 +SO-100445,2025-07-17,East,online,VE-2000,17,7650.00 +SO-100446,2025-07-17,South,online,VE-2000,6,2700.00 +SO-100447,2025-07-17,North,partner,VE-3000,1,1990.00 +SO-100448,07/17/2025,North,partner,VE-1000,8,2920.00 +SO-100449,2025-07-17,North,online,VE-3000,12,21492.00 +SO-100450,2025-07-17,North,partner,VE-2000,5,2137.50 +SO-100451,2025-07-17,South,online,VE-2000,9,4050.00 +SO-100452,2025-07-17,West,online,VE-2000,7,3150.00 +SO-100453,2025-07-17,North,partner,VE-1000,14,5110.00 +SO-100454,2025-07-17,South,partner,VE-1000,15,5475.00 +SO-100455,2025-07-17,West,partner,VE-1000,7,2299.50 +SO-100456,2025-07-17,East,direct,VE-1000,8,2920.00 +SO-100457,2025-07-17,South,online,VE-1000,4,1460.00 +SO-100458,2025-07-17,West,partner,VE-1000,1,328.50 +SO-100459,2025-07-17,North,online,VE-1000,12,4380.00 +SO-100460,2025-07-17,West,partner,VE-1000,7,2427.25 +SO-100461,2025-07-17,North,online,VE-2000,8,3600.00 +SO-100462,2025-07-17,East,partner,VE-1000,2,693.50 +SO-100463,2025-07-17,East,online,VE-2000,5,2137.50 +SO-100464,2025-07-17,East,partner,VE-2000,6,2700.00 +SO-100465,2025-07-17,East,partner,VE-2000,5,2025.00 +SO-100466,2025-07-18,South,online,VE-1000,1,365.00 +SO-100467,2025-07-18,North,online,VE-3000,17,33830.00 +SO-100468,2025-07-18,South,direct,VE-3000,6,11940.00 +SO-100469,2025-07-18,East,partner,VE-2000,1,405.00 +SO-100470,2025-07-18,East,online,VE-1000,3,985.50 +SO-100471,2025-07-18,East,partner,VE-1000,2,693.50 +SO-100472,2025-07-18,North,direct,VE-1000,10,3650.00 +SO-100473,2025-07-18,North,direct,VE-3000,7,13930.00 +SO-100474,2025-07-18,South,online,VE-1000,10,3467.50 +SO-100475,2025-07-18,North,direct,VE-1000,13,4270.50 +SO-100476,2025-07-18,South,direct,VE-2000,7,2835.00 +SO-100477,2025-07-18,South,direct,VE-2000,1,427.50 +SO-100478,2025-07-18,West,online,VE-2000,18,7290.00 +SO-100479,2025-07-18,East,partner,VE-1000,4,1460.00 +SO-100480,2025-07-18,North,direct,VE-1000,13,4507.75 +SO-100481,2025-07-18,East,partner,VE-2000,6,2700.00 +SO-100482,2025-07-18,North,online,VE-3000,8,15920.00 +SO-100483,2025-07-18,West,direct,VE-1000,16,5548.00 +SO-100484,2025-07-18,North,direct,VE-1000,6,2080.50 +SO-100485,2025-07-18,East,partner,VE-1000,5,1642.50 +SO-100486,2025-07-18,North,direct,VE-1000,10,3650.00 +SO-100487,2025-07-18,South,direct,VE-2000,11,"4,702.50" +SO-100488,2025-07-18,East,direct,VE-1000,11,4015.00 +SO-100489,2025-07-18,East,online,VE-1000,7,2555.00 +SO-100490,2025-07-18,East,online,VE-1000,1,365.00 +SO-100491,07/18/2025,North,partner,VE-1000,15,5475.00 +SO-100492,2025-07-18,South,online,VE-1000,11,3613.50 +SO-100493,2025-07-18,North,partner,VE-3000,19,37810.00 +SO-100494,2025-07-18,North,partner,VE-2000,11,4455.00 +SO-100495,2025-07-18,East,online,VE-1000,8,2774.00 +SO-100496,2025-07-18,South,online,VE-1000,16,5840.00 +SO-100497,2025-07-18,South,online,VE-1000,2,730.00 +SO-100498,2025-07-18,West,partner,VE-2000,2,900.00 +SO-100499,2025-07-18,South,online,VE-1000,20,7300.00 +SO-100500,2025-07-19,West,direct,VE-1000,12,4380.00 +SO-100501,2025-07-19,North,direct,VE-1000,17,5894.75 +SO-100502,2025-07-19,North,partner,VE-3000,18,34029.00 +SO-100503,2025-07-19,West,online,VE-1000,14,5110.00 +SO-100504,2025-07-19,South,online,VE-2000,3,1282.50 +SO-100505,2025-07-19,East,online,VE-2000,8,3600.00 +SO-100506,2025-07-19,North,direct,VE-1000,1,346.75 +SO-100507,2025-07-19,East,partner,VE-2000,14,6300.00 +SO-100508,2025-07-19,North,direct,VE-1000,14,5110.00 +SO-100509,2025-07-19,South,online,VE-1000,8,2920.00 +SO-100510,2025-07-19,North,direct,VE-1000,12,3942.00 +SO-100511,2025-07-19,West,online,VE-3000,8,14328.00 +SO-100512,2025-07-19,North,direct,VE-1000,5,1642.50 +SO-100513,2025-07-19,North,partner,VE-3000,1,1990.00 +SO-100514,2025-07-19,West,direct,VE-1000,24,8322.00 +SO-100515,2025-07-19,North,online,VE-1000,6,1971.00 +SO-100516,2025-07-19,West,direct,VE-1000,3,1095.00 +SO-100517,2025-07-19,West,online,VE-1000,1,328.50 +SO-100518,2025-07-19,South,direct,VE-2000,1,427.50 +SO-100519,2025-07-19,West,direct,VE-1000,13,4745.00 +SO-100520,2025-07-19,South,direct,VE-1000,20,6570.00 +SO-100521,2025-07-19,West,direct,VE-2000,11,4950.00 +SO-100522,2025-07-19,East,direct,VE-1000,8,2628.00 +SO-100523,2025-07-19,West,direct,VE-2000,3,1350.00 +SO-100524,2025-07-19,West,partner,VE-3000,14,27860.00 +SO-100525,2025-07-19,West,partner,VE-2000,8,3240.00 +SO-100526,2025-07-20,South,direct,VE-2000,5,2250.00 +SO-100527,2025-07-20,East,direct,VE-1000,22,8030.00 +SO-100528,2025-07-20,West,online,VE-2000,14,6300.00 +SO-100529,2025-07-20,North,partner,VE-1000,10,3650.00 +SO-100530,2025-07-20,North,online,VE-1000,10,3650.00 +SO-100531,2025-07-20,West,partner,VE-3000,18,35820.00 +SO-100532,2025-07-20,East,online,VE-3000,18,35820.00 +SO-100533,2025-07-20,North,online,VE-2000,10,4500.00 +SO-100534,2025-07-20,East,online,VE-1000,7,2555.00 +SO-100535,2025-07-20,North,partner,VE-1000,2,730.00 +SO-100536,2025-07-20,North,direct,VE-1000,6,2080.50 +SO-100537,2025-07-20,South,online,VE-1000,19,6935.00 +SO-100538,2025-07-20,North,direct,VE-3000,3,5373.00 +SO-100539,2025-07-20,North,direct,VE-2000,1,405.00 +SO-100540,2025-07-20,North,partner,VE-2000,8,3420.00 +SO-100541,2025-07-20,East,partner,VE-1000,14,4854.50 +SO-100542,2025-07-20,East,partner,VE-1000,5,"1,642.50" +SO-100543,2025-07-20,East,online,VE-1000,8,2920.00 +SO-100544,2025-07-20,North,online,VE-3000,1,1890.50 +SO-100545,2025-07-20,South,direct,VE-3000,15,26865.00 +SO-100546,2025-07-20,East,direct,VE-2000,7,2835.00 +SO-100547,2025-07-20,North,direct,VE-1000,4,"1,314.00" +SO-100548,2025-07-20,West,online,VE-2000,7,"3,150.00" +SO-100549,2025-07-20,West,online,VE-1000,17,5894.75 +SO-100550,2025-07-20,North,partner,VE-1000,4,1314.00 +SO-100551,2025-07-20,North,partner,VE-2000,20,8100.00 +SO-100552,2025-07-20,East,online,VE-2000,23,9315.00 +SO-100553,2025-07-20,North,partner,VE-1000,1,365.00 +SO-100554,2025-07-20,East,partner,VE-2000,19,8122.50 +SO-100555,2025-07-20,North,online,VE-3000,12,23880.00 +SO-100556,2025-07-20,North,online,VE-1000,9,3285.00 +SO-100557,2025-07-21,North,direct,VE-1000,11,"3,814.25" +SO-100558,2025-07-21,East,partner,VE-2000,10,4050.00 +SO-100559,2025-07-21,South,direct,VE-2000,9,4050.00 +SO-100560,2025-07-21,North,direct,VE-1000,6,2190.00 +SO-100561,2025-07-21,North,partner,VE-1000,3,1095.00 +SO-100562,2025-07-21,South,online,VE-2000,18,8100.00 +SO-100563,2025-07-21,East,online,VE-2000,13,5850.00 +SO-100564,2025-07-21,North,online,VE-3000,10,19900.00 +SO-100565,2025-07-21,North,direct,VE-1000,7,2299.50 +SO-100566,2025-07-21,South,partner,VE-3000,15,29850.00 +SO-100567,2025-07-21,East,partner,VE-3000,3,5671.50 +SO-100568,07/21/2025,West,online,VE-1000,16,5256.00 +SO-100569,2025-07-21,East,direct,VE-2000,11,4455.00 +SO-100570,2025-07-21,West,direct,VE-1000,13,4507.75 +SO-100571,2025-07-21,South,partner,VE-1000,6,2080.50 +SO-100572,2025-07-21,North,online,VE-1000,10,3650.00 +SO-100573,2025-07-21,West,partner,VE-2000,15,6750.00 +SO-100574,2025-07-21,East,direct,VE-3000,6,11940.00 +SO-100575,2025-07-21,West,direct,VE-2000,9,4050.00 +SO-100576,2025-07-21,South,online,VE-1000,7,2299.50 +SO-100577,2025-07-21,West,online,VE-1000,16,5840.00 +SO-100578,2025-07-21,South,partner,VE-2000,7,3150.00 +SO-100579,2025-07-21,West,online,VE-2000,9,4050.00 +SO-100580,2025-07-21,North,direct,VE-1000,7,2299.50 +SO-100581,2025-07-21,North,partner,VE-1000,12,3942.00 +SO-100582,2025-07-22,North,online,VE-1000,8,2920.00 +SO-100583,2025-07-22,South,partner,VE-1000,1,365.00 +SO-100584,2025-07-22,North,partner,VE-2000,11,4455.00 +SO-100585,2025-07-22,West,partner,VE-1000,15,5475.00 +SO-100586,2025-07-22,North,partner,VE-2000,3,1282.50 +SO-100587,2025-07-22,North,online,VE-1000,17,6205.00 +SO-100588,2025-07-22,East,online,VE-2000,1,450.00 +SO-100589,2025-07-22,West,partner,VE-2000,6,2700.00 +SO-100590,2025-07-22,North,partner,VE-2000,9,4050.00 +SO-100591,2025-07-22,South,direct,VE-1000,18,6570.00 +SO-100592,2025-07-22,South,partner,VE-2000,6,2700.00 +SO-100593,2025-07-22,North,online,VE-2000,8,3600.00 +SO-100594,2025-07-22,South,direct,VE-1000,16,5548.00 +SO-100595,2025-07-22,East,online,VE-2000,1,450.00 +SO-100596,2025-07-22,North,online,VE-3000,15,29850.00 +SO-100597,2025-07-22,North,online,VE-1000,12,4380.00 +SO-100598,2025-07-22,South,partner,VE-2000,7,2835.00 +SO-100599,2025-07-22,North,partner,VE-1000,2,730.00 +SO-100600,2025-07-22,North,direct,VE-1000,17,5584.50 +SO-100601,2025-07-22,North,online,VE-2000,10,4500.00 +SO-100602,2025-07-22,North,online,VE-1000,7,2555.00 +SO-100603,2025-07-22,South,direct,VE-2000,8,3600.00 +SO-100604,2025-07-22,South,direct,VE-1000,2,730.00 +SO-100605,2025-07-22,South,online,VE-3000,15,29850.00 +SO-100606,2025-07-22,South,online,VE-1000,18,"6,241.50" +SO-100607,2025-07-22,East,partner,VE-2000,2,900.00 +SO-100608,2025-07-22,South,online,VE-1000,9,3285.00 +SO-100609,2025-07-22,North,direct,VE-1000,16,5256.00 +SO-100610,2025-07-22,South,partner,VE-1000,21,7281.75 +SO-100611,2025-07-23,East,online,VE-2000,10,4500.00 +SO-100612,2025-07-23,South,partner,VE-1000,18,6570.00 +SO-100613,2025-07-23,East,online,VE-1000,13,4270.50 +SO-100614,2025-07-23,North,partner,VE-1000,7,2555.00 +SO-100615,2025-07-23,East,direct,VE-1000,15,5475.00 +SO-100616,2025-07-23,North,partner,VE-3000,15,"29,850.00" +SO-100617,2025-07-23,South,partner,VE-3000,6,11940.00 +SO-100618,2025-07-23,West,partner,VE-1000,7,2299.50 +SO-100619,2025-07-23,North,online,VE-1000,14,4599.00 +SO-100620,2025-07-23,East,partner,VE-3000,9,17910.00 +SO-100621,2025-07-23,North,direct,VE-1000,8,2920.00 +SO-100622,2025-07-23,South,online,VE-2000,1,450.00 +SO-100623,2025-07-23,East,direct,VE-1000,14,5110.00 +SO-100624,2025-07-23,North,online,VE-1000,8,2920.00 +SO-100625,2025-07-23,West,online,VE-1000,1,365.00 +SO-100626,2025-07-23,North,online,VE-2000,14,5670.00 +SO-100627,2025-07-23,West,online,VE-1000,12,4380.00 +SO-100628,2025-07-23,North,online,VE-1000,2,730.00 +SO-100629,2025-07-23,South,direct,VE-1000,9,"3,120.75" +SO-100630,2025-07-23,North,direct,VE-2000,5,2250.00 +SO-100631,2025-07-23,South,partner,VE-1000,20,7300.00 +SO-100632,2025-07-23,East,online,VE-2000,20,8100.00 +SO-100633,2025-07-23,South,direct,VE-2000,1,450.00 +SO-100634,2025-07-23,North,direct,VE-1000,16,5840.00 +SO-100635,2025-07-23,West,online,VE-1000,1,365.00 +SO-100636,2025-07-23,North,partner,VE-2000,5,2250.00 +SO-100637,2025-07-24,North,online,VE-1000,11,4015.00 +SO-100638,2025-07-24,South,partner,VE-3000,12,23880.00 +SO-100639,2025-07-24,North,partner,VE-1000,20,7300.00 +SO-100640,2025-07-24,South,partner,VE-1000,7,2299.50 +SO-100641,2025-07-24,North,partner,VE-2000,16,7200.00 +SO-100642,2025-07-24,East,partner,VE-3000,13,24576.50 +SO-100643,2025-07-24,North,online,VE-1000,11,3814.25 +SO-100644,2025-07-24,East,partner,VE-1000,11,4015.00 +SO-100645,2025-07-24,North,direct,VE-1000,7,2427.25 +SO-100646,2025-07-24,North,online,VE-1000,7,2299.50 +SO-100647,2025-07-24,East,partner,VE-3000,11,21890.00 +SO-100648,2025-07-24,East,direct,VE-1000,5,1825.00 +SO-100649,2025-07-24,West,direct,VE-1000,9,3120.75 +SO-100650,2025-07-24,East,online,VE-1000,10,3467.50 +SO-100651,2025-07-24,South,online,VE-3000,1,1791.00 +SO-100652,2025-07-24,North,partner,VE-1000,10,3285.00 +SO-100653,2025-07-24,East,direct,VE-2000,17,6885.00 +SO-100654,2025-07-24,North,partner,VE-1000,5,1825.00 +SO-100655,2025-07-24,West,direct,VE-2000,7,3150.00 +SO-100656,2025-07-24,West,direct,VE-2000,17,6885.00 +SO-100657,2025-07-24,West,online,VE-3000,1,1990.00 +SO-100658,2025-07-24,West,direct,VE-1000,10,3467.50 +SO-100659,2025-07-24,South,direct,VE-1000,17,5894.75 +SO-100660,2025-07-24,North,direct,VE-3000,4,7960.00 +SO-100661,2025-07-24,South,direct,VE-3000,9,17910.00 +SO-100662,2025-07-24,South,direct,VE-3000,8,15920.00 +SO-100663,2025-07-24,East,online,VE-2000,8,3600.00 +SO-100664,2025-07-25,North,online,VE-2000,4,1710.00 +SO-100665,2025-07-25,South,direct,VE-2000,3,1215.00 +SO-100666,2025-07-25,South,direct,VE-3000,5,8955.00 +SO-100667,2025-07-25,West,partner,VE-1000,6,2190.00 +SO-100668,2025-07-25,South,partner,VE-2000,17,6885.00 +SO-100669,2025-07-25,West,partner,VE-2000,10,4500.00 +SO-100670,2025-07-25,South,partner,VE-1000,17,5584.50 +SO-100671,2025-07-25,East,direct,VE-1000,19,6935.00 +SO-100672,2025-07-25,South,online,VE-1000,10,3650.00 +SO-100673,2025-07-25,East,direct,VE-1000,13,4745.00 +SO-100674,2025-07-25,South,partner,VE-1000,5,1642.50 +SO-100675,2025-07-25,East,direct,VE-1000,8,2628.00 +SO-100676,2025-07-25,North,direct,VE-2000,13,5265.00 +SO-100677,2025-07-25,South,online,VE-2000,16,7200.00 +SO-100678,2025-07-25,South,online,VE-2000,5,2250.00 +SO-100679,2025-07-25,North,partner,VE-2000,1,427.50 +SO-100680,2025-07-25,North,partner,VE-1000,12,4380.00 +SO-100681,2025-07-25,South,direct,VE-1000,11,3613.50 +SO-100682,2025-07-25,North,partner,VE-2000,6,2700.00 +SO-100683,2025-07-25,South,partner,VE-1000,1,365.00 +SO-100684,2025-07-25,East,online,VE-1000,16,5840.00 +SO-100685,2025-07-25,South,online,VE-1000,2,730.00 +SO-100686,2025-07-25,South,partner,VE-1000,9,3285.00 +SO-100687,2025-07-25,South,online,VE-2000,12,4860.00 +SO-100688,2025-07-25,East,direct,VE-1000,11,4015.00 +SO-100689,2025-07-25,North,online,VE-1000,5,1825.00 +SO-100690,2025-07-25,South,direct,VE-1000,7,2427.25 +SO-100691,2025-07-25,South,direct,VE-3000,9,16119.00 +SO-100692,07/26/2025,South,partner,VE-1000,16,5840.00 +SO-100693,2025-07-26,North,direct,VE-1000,2,730.00 +SO-100694,2025-07-26,West,online,VE-2000,2,855.00 +SO-100695,2025-07-26,South,partner,VE-1000,8,2774.00 +SO-100696,07/26/2025,South,partner,VE-3000,13,25870.00 +SO-100697,2025-07-26,North,partner,VE-2000,14,5985.00 +SO-100698,2025-07-26,North,partner,VE-2000,3,1282.50 +SO-100699,2025-07-26,North,partner,VE-2000,8,3420.00 +SO-100700,2025-07-26,East,partner,VE-2000,10,4500.00 +SO-100701,2025-07-26,East,direct,VE-2000,3,1350.00 +SO-100702,2025-07-26,West,direct,VE-1000,1,365.00 +SO-100703,2025-07-26,South,partner,VE-2000,14,5670.00 +SO-100704,2025-07-26,North,partner,VE-2000,12,5400.00 +SO-100705,2025-07-26,North,online,VE-1000,14,4854.50 +SO-100706,2025-07-26,North,partner,VE-3000,15,26865.00 +SO-100707,2025-07-26,West,partner,VE-1000,10,3650.00 +SO-100708,2025-07-26,South,direct,VE-2000,9,4050.00 +SO-100709,2025-07-26,West,direct,VE-1000,19,6935.00 +SO-100710,2025-07-26,West,partner,VE-2000,4,1710.00 +SO-100711,07/26/2025,North,online,VE-3000,2,3781.00 +SO-100712,2025-07-26,West,online,VE-2000,7,3150.00 +SO-100713,2025-07-26,South,partner,VE-2000,15,6750.00 +SO-100714,2025-07-26,South,online,VE-2000,12,5400.00 +SO-100715,2025-07-26,South,partner,VE-1000,11,3814.25 +SO-100716,2025-07-26,North,partner,VE-2000,6,2700.00 +SO-100717,2025-07-26,East,partner,VE-2000,10,4050.00 +SO-100718,2025-07-26,East,online,VE-2000,6,2700.00 +SO-100719,2025-07-26,West,direct,VE-1000,22,7628.50 +SO-100720,2025-07-26,North,partner,VE-2000,20,8100.00 +SO-100721,2025-07-26,East,online,VE-1000,4,1460.00 +SO-100722,2025-07-26,East,online,VE-2000,4,1620.00 +SO-100723,2025-07-26,South,online,VE-3000,1,1890.50 +SO-100724,2025-07-26,North,partner,VE-3000,9,17014.50 +SO-100725,2025-07-26,North,online,VE-1000,1,365.00 +SO-100726,2025-07-26,South,online,VE-1000,1,365.00 +SO-100727,2025-07-26,North,direct,VE-3000,6,10746.00 +SO-100728,2025-07-27,South,direct,VE-3000,13,25870.00 +SO-100729,2025-07-27,North,partner,VE-1000,4,1460.00 +SO-100730,2025-07-27,South,direct,VE-2000,14,6300.00 +SO-100731,2025-07-27,East,direct,VE-2000,16,6480.00 +SO-100732,2025-07-27,West,partner,VE-1000,10,3650.00 +SO-100733,2025-07-27,North,online,VE-1000,3,1095.00 +SO-100734,2025-07-27,North,partner,VE-3000,7,13233.50 +SO-100735,2025-07-27,East,partner,VE-3000,5,9452.50 +SO-100736,2025-07-27,South,direct,VE-3000,6,10746.00 +SO-100737,2025-07-27,West,direct,VE-2000,1,450.00 +SO-100738,2025-07-27,South,online,VE-1000,11,4015.00 +SO-100739,2025-07-27,East,online,VE-1000,10,3467.50 +SO-100740,2025-07-27,West,partner,VE-2000,11,4950.00 +SO-100741,2025-07-27,West,direct,VE-2000,6,2565.00 +SO-100742,07/27/2025,North,direct,VE-1000,10,3650.00 +SO-100743,2025-07-27,East,direct,VE-2000,6,2700.00 +SO-100744,2025-07-27,North,online,VE-1000,1,365.00 +SO-100745,2025-07-27,South,online,VE-1000,11,3613.50 +SO-100746,2025-07-27,South,online,VE-1000,4,1460.00 +SO-100747,2025-07-27,North,direct,VE-2000,1,450.00 +SO-100748,2025-07-27,North,online,VE-2000,8,3600.00 +SO-100749,2025-07-27,East,online,VE-1000,9,3120.75 +SO-100750,2025-07-27,West,direct,VE-2000,6,2430.00 +SO-100751,2025-07-28,East,partner,VE-1000,20,6935.00 +SO-100752,2025-07-28,East,partner,VE-1000,14,4599.00 +SO-100753,2025-07-28,East,partner,VE-2000,14,6300.00 +SO-100754,2025-07-28,East,direct,VE-3000,9,17910.00 +SO-100755,2025-07-28,North,partner,VE-1000,5,"1,825.00" +SO-100756,2025-07-28,East,online,VE-3000,3,5373.00 +SO-100757,2025-07-28,East,partner,VE-1000,16,5840.00 +SO-100758,2025-07-28,East,partner,VE-1000,13,4745.00 +SO-100759,2025-07-28,North,online,VE-1000,4,1460.00 +SO-100760,2025-07-28,West,partner,VE-2000,1,405.00 +SO-100761,2025-07-28,South,direct,VE-2000,8,3600.00 +SO-100762,2025-07-28,North,partner,VE-2000,15,6075.00 +SO-100763,2025-07-28,North,direct,VE-1000,15,"5,475.00" +SO-100764,2025-07-28,North,partner,VE-3000,5,8955.00 +SO-100765,2025-07-28,East,partner,VE-3000,6,11940.00 +SO-100766,2025-07-28,East,direct,VE-1000,9,3285.00 +SO-100767,2025-07-28,South,online,VE-1000,22,8030.00 +SO-100768,2025-07-28,North,online,VE-1000,11,3613.50 +SO-100769,2025-07-28,South,direct,VE-1000,8,2628.00 +SO-100770,2025-07-28,North,partner,VE-2000,10,4050.00 +SO-100771,2025-07-28,East,online,VE-1000,14,5110.00 +SO-100772,2025-07-28,North,partner,VE-1000,13,4745.00 +SO-100773,2025-07-28,East,partner,VE-2000,5,2250.00 +SO-100774,2025-07-28,West,online,VE-1000,6,2190.00 +SO-100775,2025-07-28,East,direct,VE-1000,6,2080.50 +SO-100776,2025-07-28,West,online,VE-3000,17,30447.00 +SO-100777,2025-07-29,South,online,VE-2000,10,4050.00 +SO-100778,2025-07-29,North,online,VE-2000,14,6300.00 +SO-100779,2025-07-29,North,online,VE-1000,13,4745.00 +SO-100780,2025-07-29,West,partner,VE-1000,10,3650.00 +SO-100781,2025-07-29,South,partner,VE-1000,11,4015.00 +SO-100782,2025-07-29,South,partner,VE-3000,1,1990.00 +SO-100783,2025-07-29,North,online,VE-1000,7,2427.25 +SO-100784,2025-07-29,South,online,VE-1000,6,2190.00 +SO-100785,2025-07-29,South,online,VE-2000,1,427.50 +SO-100786,2025-07-29,North,direct,VE-3000,9,16119.00 +SO-100787,2025-07-29,North,partner,VE-1000,7,2427.25 +SO-100788,2025-07-29,East,online,VE-1000,14,4854.50 +SO-100789,2025-07-29,North,direct,VE-3000,1,1990.00 +SO-100790,2025-07-29,South,partner,VE-1000,1,328.50 +SO-100791,2025-07-29,South,online,VE-1000,9,3120.75 +SO-100792,2025-07-29,South,direct,VE-3000,7,13930.00 +SO-100793,2025-07-29,East,partner,VE-2000,8,3420.00 +SO-100794,2025-07-29,East,partner,VE-1000,8,2920.00 +SO-100795,2025-07-29,South,online,VE-1000,5,1642.50 +SO-100796,2025-07-29,South,online,VE-2000,6,2700.00 +SO-100797,2025-07-29,North,online,VE-3000,7,12537.00 +SO-100798,2025-07-29,West,direct,VE-2000,6,2430.00 +SO-100799,2025-07-29,South,online,VE-2000,12,5400.00 +SO-100800,2025-07-29,South,online,VE-2000,7,3150.00 +SO-100801,2025-07-29,West,online,VE-3000,13,25870.00 +SO-100802,2025-07-29,South,online,VE-1000,8,2628.00 +SO-100803,2025-07-29,South,partner,VE-1000,9,2956.50 +SO-100804,2025-07-30,East,direct,VE-2000,6,2565.00 +SO-100805,2025-07-30,West,partner,VE-3000,8,15124.00 +SO-100806,2025-07-30,North,partner,VE-1000,9,2956.50 +SO-100807,2025-07-30,West,partner,VE-1000,1,365.00 +SO-100808,2025-07-30,North,partner,VE-1000,8,2920.00 +SO-100809,2025-07-30,North,direct,VE-2000,11,4950.00 +SO-100810,2025-07-30,East,partner,VE-1000,3,985.50 +SO-100811,2025-07-30,East,partner,VE-1000,7,2555.00 +SO-100812,2025-07-30,EAST,online,VE-3000,4,7960.00 +SO-100813,2025-07-30,South,online,VE-3000,3,5373.00 +SO-100814,2025-07-30,North,direct,VE-1000,10,3650.00 +SO-100815,2025-07-30,North,direct,VE-2000,16,7200.00 +SO-100816,2025-07-30,North,direct,VE-1000,3,1095.00 +SO-100817,2025-07-30,North,partner,VE-1000,7,2555.00 +SO-100818,2025-07-30,South,online,VE-1000,1,365.00 +SO-100819,2025-07-30,South,direct,VE-2000,12,5400.00 +SO-100820,2025-07-30,South,partner,VE-1000,1,346.75 +SO-100821,2025-07-30,West,direct,VE-1000,2,693.50 +SO-100822,2025-07-30,South,direct,VE-1000,13,4745.00 +SO-100823,2025-07-30,North,direct,VE-2000,1,427.50 +SO-100824,2025-07-30,South,direct,VE-1000,15,5201.25 +SO-100825,2025-07-30,West,partner,VE-1000,9,3120.75 +SO-100826,2025-07-30,North,direct,VE-1000,11,3613.50 +SO-100827,2025-07-30,East,partner,VE-3000,15,29850.00 +SO-100828,2025-07-30,North,partner,VE-2000,6,2565.00 +SO-100829,2025-07-30,North,direct,VE-1000,11,"4,015.00" +SO-100830,2025-07-30,East,direct,VE-1000,19,6935.00 +SO-100831,2025-07-30,North,online,VE-1000,15,5475.00 +SO-100832,2025-07-30,North,direct,VE-2000,11,4950.00 +SO-100833,2025-07-30,South,direct,VE-2000,2,810.00 +SO-100834,2025-07-31,North,direct,VE-3000,4,7960.00 +SO-100835,2025-07-31,East,direct,VE-2000,1,450.00 +SO-100836,2025-07-31,South,partner,VE-3000,19,37810.00 +SO-100837,2025-07-31,West,direct,VE-1000,5,1733.75 +SO-100838,2025-07-31,South,direct,VE-2000,10,4500.00 +SO-100839,2025-07-31,South,partner,VE-1000,6,2080.50 +SO-100840,2025-07-31,South,partner,VE-2000,7,2992.50 +SO-100841,2025-07-31,South,direct,VE-3000,9,17910.00 +SO-100842,2025-07-31,South,direct,VE-1000,1,328.50 +SO-100843,2025-07-31,West,partner,VE-2000,13,5850.00 +SO-100844,2025-07-31,South,direct,VE-1000,11,3814.25 +SO-100845,2025-07-31,East,online,VE-2000,14,5985.00 +SO-100846,2025-07-31,North,online,VE-2000,13,5850.00 +SO-100847,2025-07-31,North,direct,VE-2000,11,"4,702.50" +SO-100848,2025-07-31,South,online,VE-2000,12,5400.00 +SO-100849,2025-07-31,South,direct,VE-1000,5,1733.75 +SO-100850,2025-07-31,North,online,VE-1000,11,4015.00 +SO-100851,2025-07-31,South,partner,VE-1000,12,4380.00 +SO-100852,2025-07-31,North,online,VE-2000,3,1350.00 +SO-100853,2025-07-31,West,partner,VE-1000,10,3285.00 +SO-100854,2025-07-31,South,online,VE-2000,9,3847.50 +SO-100855,2025-07-31,South,partner,VE-1000,7,2427.25 +SO-100856,2025-07-31,South,online,VE-2000,17,7650.00 +SO-100857,2025-07-31,East,online,VE-2000,1,450.00 +SO-100858,2025-07-31,West,partner,VE-2000,14,6300.00 +SO-100859,2025-07-31,North,partner,VE-3000,14,27860.00 +SO-100860,2025-07-31,North,online,VE-1000,1,328.50 +SO-100861,2025-07-31,North,direct,VE-1000,14,5110.00 +SO-100862,2025-08-01,South,partner,VE-3000,18,32238.00 +SO-100863,2025-08-01,East,direct,VE-2000,11,4455.00 +SO-100864,2025-08-01,North,online,VE-3000,1,1791.00 +SO-100865,2025-08-01,South,direct,VE-1000,7,"2,555.00" +SO-100866,2025-08-01,West,online,VE-1000,5,1825.00 +SO-100867,2025-08-01,South,direct,VE-2000,13,5850.00 +SO-100868,2025-08-01,North,online,VE-2000,3,1282.50 +SO-100869,2025-08-01,South,online,VE-1000,12,3942.00 +SO-100870,2025-08-01,North,partner,VE-3000,3,5970.00 +SO-100871,2025-08-01,West,direct,VE-2000,9,4050.00 +SO-100872,2025-08-01,West,online,VE-3000,10,19900.00 +SO-100873,2025-08-01,North,online,VE-1000,24,8760.00 +SO-100874,2025-08-01,North,direct,VE-2000,8, +SO-100875,2025-08-01,East,partner,VE-1000,10,3285.00 +SO-100876,2025-08-01,South,direct,VE-3000,12,23880.00 +SO-100877,2025-08-01,South,online,VE-3000,3,5970.00 +SO-100878,2025-08-01,North,online,VE-1000,7,2299.50 +SO-100879,2025-08-01,South,online,VE-3000,13,25870.00 +SO-100880,2025-08-01,West,online,VE-3000,22,43780.00 +SO-100881,2025-08-01,South,partner,VE-1000,4,1460.00 +SO-100882,2025-08-01,North,online,VE-2000,21,9450.00 +SO-100883,2025-08-01,East,direct,VE-2000,4,1800.00 +SO-100884,2025-08-01,South,partner,VE-1000,1,365.00 +SO-100885,2025-08-01,South,partner,VE-3000,10,19900.00 +SO-100886,2025-08-01,East,partner,VE-1000,3,1095.00 +SO-100887,2025-08-01,South,partner,VE-1000,1,365.00 +SO-100888,2025-08-01,North,direct,VE-1000,10,3650.00 +SO-100889,2025-08-02,East,online,VE-2000,16,6480.00 +SO-100890,2025-08-02,North,online,VE-3000,4,7562.00 +SO-100891,2025-08-02,North,partner,VE-1000,10,3650.00 +SO-100892,2025-08-02,East,online,VE-1000,1,346.75 +SO-100893,2025-08-02,South,online,VE-1000,10,3650.00 +SO-100894,2025-08-02,North,direct,VE-1000,10,3650.00 +SO-100895,2025-08-02,South,direct,VE-2000,4,1800.00 +SO-100896,2025-08-02,North,partner,VE-1000,2,730.00 +SO-100897,2025-08-02,North,partner,VE-2000,10,4500.00 +SO-100898,2025-08-02,East,online,VE-2000,4,1620.00 +SO-100899,2025-08-02,South,direct,VE-1000,15,5475.00 +SO-100900,2025-08-02,South,direct,VE-1000,3,1040.25 +SO-100901,2025-08-02,South,partner,VE-2000,5,2250.00 +SO-100902,2025-08-02,North,direct,VE-1000,9,3285.00 +SO-100903,2025-08-02,North,partner,VE-1000,8,2774.00 +SO-100904,2025-08-02,East,online,VE-2000,6,2700.00 +SO-100905,2025-08-02,South,direct,VE-2000,1,450.00 +SO-100906,2025-08-02,West,direct,VE-3000,8,15920.00 +SO-100907,2025-08-02,East,online,VE-1000,6,2190.00 +SO-100908,2025-08-03,East,direct,VE-1000,2,730.00 +SO-100909,2025-08-03,West,partner,VE-1000,18,6570.00 +SO-100910,2025-08-03,South,online,VE-2000,9,4050.00 +SO-100911,2025-08-03,North,online,VE-2000,6,2700.00 +SO-100912,2025-08-03,North,online,VE-2000,11,4702.50 +SO-100913,2025-08-03,East,partner,VE-2000,6,2700.00 +SO-100914,2025-08-03,North,partner,VE-1000,15,5201.25 +SO-100915,2025-08-03,East,partner,VE-1000,9,2956.50 +SO-100916,2025-08-03,North,online,VE-2000,16,7200.00 +SO-100917,2025-08-03,East,partner,VE-2000,6,2700.00 +SO-100918,2025-08-03,North,partner,VE-1000,10,3467.50 +SO-100919,2025-08-03,West,online,VE-3000,9,17910.00 +SO-100920,2025-08-03,South,direct,VE-3000,10,19900.00 +SO-100921,2025-08-03,East,direct,VE-2000,21,8505.00 +SO-100922,2025-08-03,North,online,VE-1000,7,2555.00 +SO-100923,2025-08-03,North,partner,VE-2000,6,2700.00 +SO-100924,2025-08-03,West,direct,VE-2000,6,2430.00 +SO-100925,2025-08-03,West,partner,VE-1000,3,985.50 +SO-100926,2025-08-03,East,direct,VE-1000,8,2920.00 +SO-100927,2025-08-03,West,online,VE-1000,23,7555.50 +SO-100928,2025-08-03,North,online,VE-1000,8,2774.00 +SO-100929,2025-08-03,East,online,VE-2000,9,4050.00 +SO-100930,2025-08-03,North,online,VE-1000,13,4745.00 +SO-100931,2025-08-03,North,partner,VE-1000,1,365.00 +SO-100932,2025-08-03,South,direct,VE-2000,5,2250.00 +SO-100933,2025-08-03,West,online,VE-1000,11,3814.25 +SO-100934,2025-08-03,North,online,VE-3000,6,11940.00 +SO-100935,2025-08-03,East,direct,VE-1000,24,8760.00 +SO-100936,2025-08-03,West,partner,VE-1000,12,"4,161.00" +SO-100937,2025-08-03,North,online,VE-1000,5,1642.50 +SO-100938,2025-08-03,East,online,VE-1000,24,8760.00 +SO-100939,2025-08-03,South,online,VE-2000,8,3420.00 +SO-100940,2025-08-03,South,partner,VE-1000,5,1825.00 +SO-100941,2025-08-03,East,direct,VE-1000,13,4507.75 +SO-100942,2025-08-03,North,partner,VE-1000,1,365.00 +SO-100943,2025-08-03,South,partner,VE-1000,9,2956.50 +SO-100944,2025-08-03,North,online,VE-2000,8,3420.00 +SO-100945,2025-08-03,East,online,VE-1000,10,3285.00 +SO-100946,2025-08-03,South,direct,VE-2000,7,3150.00 +SO-100947,2025-08-03,South,online,VE-1000,7,2555.00 +SO-100948,2025-08-03,South,online,VE-2000,16,7200.00 +SO-100949,2025-08-03,East,online,VE-2000,17,7650.00 +SO-100950,2025-08-04, West,direct,VE-1000,5,1825.00 +SO-100951,2025-08-04,North,direct,VE-1000,2,730.00 +SO-100952,2025-08-04,North,direct,VE-1000,2,730.00 +SO-100953,2025-08-04,East,direct,VE-3000,2,3980.00 +SO-100954,2025-08-04,South,online,VE-1000,4,1460.00 +SO-100955,2025-08-04,West,online,VE-1000,17,5894.75 +SO-100956,2025-08-04,South,partner,VE-1000,21,7665.00 +SO-100957,2025-08-04,West,online,VE-1000,4,1460.00 +SO-100958,2025-08-04,South,direct,VE-1000,14,4854.50 +SO-100959,2025-08-04,South,online,VE-3000,16,31840.00 +SO-100960,2025-08-04,East,direct,VE-2000,10,4500.00 +SO-100961,2025-08-04,South,direct,VE-1000,12,4380.00 +SO-100962,2025-08-04,South,partner,VE-1000,17,6205.00 +SO-100963,2025-08-04,East,direct,VE-3000,6,10746.00 +SO-100964,2025-08-04,West,partner,VE-1000,3,1095.00 +SO-100965,2025-08-04,East,direct,VE-2000,13,5265.00 +SO-100966,2025-08-04,east,online,VE-1000,20,7300.00 +SO-100967,2025-08-04,North,partner,VE-2000,7,3150.00 +SO-100968,2025-08-04,East,partner,VE-2000,5,2250.00 +SO-100969,2025-08-04,West,online,VE-1000,8,2628.00 +SO-100970,2025-08-04,East,direct,VE-1000,5,1733.75 +SO-100971,2025-08-04,North,direct,VE-3000,1,1890.50 +SO-100972,2025-08-04,West,partner,VE-1000,14, +SO-100973,2025-08-04,East,partner,VE-2000,3,1282.50 +SO-100974,2025-08-05,East,online,VE-2000,14,6300.00 +SO-100975,2025-08-05,South,partner,VE-2000,13,5850.00 +SO-100976,2025-08-05,South,online,VE-1000,9,2956.50 +SO-100977,2025-08-05,East,online,VE-1000,18,6241.50 +SO-100978,2025-08-05,West,partner,VE-1000,16,5840.00 +SO-100979,2025-08-05,East,direct,VE-1000,6,"1,971.00" +SO-100980,2025-08-05,West,online,VE-1000,18,6570.00 +SO-100981,2025-08-05,North,partner,VE-1000,4,1460.00 +SO-100982,2025-08-05,East,direct,VE-3000,13,25870.00 +SO-100983,2025-08-05,East,direct,VE-3000,7,12537.00 +SO-100984,2025-08-05,West,partner,VE-1000,10,3285.00 +SO-100985,2025-08-05,East,direct,VE-3000,3,5970.00 +SO-100986,2025-08-05,South,online,VE-1000,15,5475.00 +SO-100987,2025-08-05,East,online,VE-2000,17,6885.00 +SO-100988,2025-08-05,East,partner,VE-3000,6,11940.00 +SO-100989,2025-08-05,North,partner,VE-2000,6,2700.00 +SO-100990,2025-08-05,North,online,VE-2000,12,5400.00 +SO-100991,2025-08-05,South,direct,VE-1000,14,5110.00 +SO-100992,2025-08-05,North,direct,VE-3000,1,1990.00 +SO-100993,2025-08-05,North,online,VE-3000,15,29850.00 +SO-100994,2025-08-05,East,direct,VE-2000,7,3150.00 +SO-100995,2025-08-05,South,online,VE-3000,4,7960.00 +SO-100996,2025-08-05,East,direct,VE-1000,1,365.00 +SO-100997,2025-08-05,South,partner,VE-3000,17,33830.00 +SO-100998,2025-08-05,North,direct,VE-2000,3,1350.00 +SO-100999,2025-08-05,South,direct,VE-3000,2,3781.00 +SO-101000,2025-08-06,East,partner,VE-1000,5,1825.00 +SO-101001,2025-08-06,East,online,VE-3000,12,22686.00 +SO-101002,2025-08-06,North,direct,VE-1000,1,346.75 +SO-101003,2025-08-06,North,online,VE-1000,10,3467.50 +SO-101004,2025-08-06,North,online,VE-1000,4,1314.00 +SO-101005,2025-08-06,South,direct,VE-3000,8,14328.00 +SO-101006,2025-08-06,West,online,VE-1000,4,1460.00 +SO-101007,2025-08-06,West,partner,VE-1000,1,346.75 +SO-101008,2025-08-06,West,partner,VE-1000,3,985.50 +SO-101009,2025-08-06,East,direct,VE-3000,3,5970.00 +SO-101010,2025-08-06,East,partner,VE-2000,7,3150.00 +SO-101011,2025-08-06,East,online,VE-1000,14,4599.00 +SO-101012,2025-08-06,West,direct,VE-1000,7,2299.50 +SO-101013,2025-08-06,South,online,VE-1000,10,3650.00 +SO-101014,2025-08-06,North,partner,VE-3000,4,7960.00 +SO-101015,2025-08-06,South,partner,VE-1000,17,6205.00 +SO-101016,2025-08-06,South,direct,VE-1000,15,5475.00 +SO-101017,2025-08-06,West,direct,VE-3000,10,19900.00 +SO-101018,2025-08-06,South,direct,VE-1000,7,2555.00 +SO-101019,2025-08-06,North,online,VE-3000,13,25870.00 +SO-101020,2025-08-06,East,online,VE-1000,16,5548.00 +SO-101021,2025-08-06,South,partner,VE-2000,17,7267.50 +SO-101022,2025-08-06,North,online,VE-2000,9,3645.00 +SO-101023,2025-08-06,West,direct,VE-1000,13,4507.75 +SO-101024,2025-08-06,North,partner,VE-1000,7,2555.00 +SO-101025,2025-08-06,North,direct,VE-1000,9,2956.50 +SO-101026,2025-08-06,East,online,VE-1000,4,1460.00 +SO-101027,2025-08-07,East,partner,VE-1000,15,5475.00 +SO-101028,2025-08-07,West,direct,VE-1000,1,328.50 +SO-101029,2025-08-07,West,online,VE-2000,15,6412.50 +SO-101030,2025-08-07,West,direct,VE-3000,10,19900.00 +SO-101031,2025-08-07,South,online,VE-2000,15,6750.00 +SO-101032,2025-08-07,North,partner,VE-2000,18,7695.00 +SO-101033,2025-08-07,North,direct,VE-1000,3,1095.00 +SO-101034,2025-08-07,South,direct,VE-1000,11,4015.00 +SO-101035,2025-08-07,East,partner,VE-3000,5,9950.00 +SO-101036,2025-08-07,South,partner,VE-1000,19,6935.00 +SO-101037,08/07/2025,North,direct,VE-1000,10,3467.50 +SO-101038,2025-08-07,East,direct,VE-1000,6,2190.00 +SO-101039,2025-08-07,North,direct,VE-2000,8,3600.00 +SO-101040,2025-08-07,South,partner,VE-1000,1,365.00 +SO-101041,2025-08-07,North,online,VE-3000,3,5373.00 +SO-101042,2025-08-07,South,partner,VE-2000,5,2250.00 +SO-101043,2025-08-07,North,partner,VE-2000,8,3600.00 +SO-101044,2025-08-07,West,partner,VE-2000,22,9900.00 +SO-101045,2025-08-07,East,online,VE-1000,16,5256.00 +SO-101046,2025-08-07,EAST,direct,VE-1000,1,346.75 +SO-101047,2025-08-07,South,direct,VE-1000,9,3285.00 +SO-101048,2025-08-07,West,partner,VE-2000,3,1282.50 +SO-101049,2025-08-08,West,online,VE-3000,1,1890.50 +SO-101050,2025-08-08,West,online,VE-1000,7,2299.50 +SO-101051,2025-08-08,East,online,VE-3000,12,23880.00 +SO-101052,2025-08-08,North,direct,VE-2000,1,450.00 +SO-101053,2025-08-08,South,direct,VE-2000,8,3600.00 +SO-101054,2025-08-08,East,online,VE-1000,9,3120.75 +SO-101055,2025-08-08,North,partner,VE-2000,3,1350.00 +SO-101056,2025-08-08,East,partner,VE-2000,11,4950.00 +SO-101057,2025-08-08,East,direct,VE-1000,16,5840.00 +SO-101058,2025-08-08,East,partner,VE-2000,5,2250.00 +SO-101059,2025-08-08,South,online,VE-3000,8,15920.00 +SO-101060,2025-08-08,South,online,VE-1000,10,3650.00 +SO-101061,2025-08-08,East,partner,VE-1000,13,4270.50 +SO-101062,2025-08-08,South,direct,VE-1000,6,2190.00 +SO-101063,2025-08-08,South,partner,VE-3000,5,8955.00 +SO-101064,2025-08-08,West,online,VE-2000,14,6300.00 +SO-101065,2025-08-08,West,partner,VE-1000,1,365.00 +SO-101066,2025-08-08,South,online,VE-1000,13,4745.00 +SO-101067,2025-08-08,East,partner,VE-2000,7,3150.00 +SO-101068,2025-08-08,East,partner,VE-1000,7,2427.25 +SO-101069,2025-08-08,North,direct,VE-1000,3,1095.00 +SO-101070,2025-08-08,East,partner,VE-3000,4,7562.00 +SO-101071,2025-08-08,West,direct,VE-1000,14,5110.00 +SO-101072,2025-08-08,East,direct,VE-1000,1,365.00 +SO-101073,2025-08-09,South,direct,VE-2000,11,4950.00 +SO-101074,2025-08-09,North,direct,VE-2000,9,3847.50 +SO-101075,2025-08-09,South,online,VE-1000,11,3613.50 +SO-101076,2025-08-09,South,online,VE-1000,6,2190.00 +SO-101077,2025-08-09,West,partner,VE-1000,9,2956.50 +SO-101078,2025-08-09,East,partner,VE-1000,5,1825.00 +SO-101079,2025-08-09,North,online,VE-2000,6,2565.00 +SO-101080,2025-08-09,West,partner,VE-2000,8,3420.00 +SO-101081,2025-08-09,North,direct,VE-1000,12,4380.00 +SO-101082,2025-08-09,South,partner,VE-1000,7,2299.50 +SO-101083,2025-08-09,North,direct,VE-1000,11,4015.00 +SO-101084,2025-08-09,South,online,VE-2000,7,2992.50 +SO-101085,2025-08-09,South,direct,VE-2000,6,2430.00 +SO-101086,2025-08-09,East,direct,VE-2000,8,3600.00 +SO-101087,2025-08-09,West,online,VE-1000,10,3467.50 +SO-101088,2025-08-09,North,direct,VE-2000,8,3240.00 +SO-101089,2025-08-09,North,direct,VE-1000,11,4015.00 +SO-101090,2025-08-09,West,partner,VE-2000,14,6300.00 +SO-101091,2025-08-09,East,partner,VE-1000,10,3650.00 +SO-101092,2025-08-09,South,direct,VE-2000,9,3645.00 +SO-101093,2025-08-09,North,online,VE-2000,13,5850.00 +SO-101094,2025-08-09,North,online,VE-1000,12,4380.00 +SO-101095,2025-08-09,South,direct,VE-1000,14,5110.00 +SO-101096,2025-08-10,East,direct,VE-3000,2,3781.00 +SO-101097,08/10/2025,West,direct,VE-2000,12,5400.00 +SO-101098,2025-08-10,East,direct,VE-1000,12,4380.00 +SO-101099,2025-08-10,North,partner,VE-2000,10,4275.00 +SO-101100,2025-08-10,South,direct,VE-1000,11,4015.00 +SO-101101,2025-08-10,South,partner,VE-1000,5,1733.75 +SO-101102,2025-08-10,North,online,VE-1000,7,2555.00 +SO-101103,2025-08-10,West,partner,VE-2000,11,4950.00 +SO-101104,2025-08-10,West,direct,VE-2000,12,5130.00 +SO-101105,2025-08-10,North,partner,VE-2000,5,2137.50 +SO-101106,2025-08-10,South,online,VE-3000,12,23880.00 +SO-101107,2025-08-10,South,direct,VE-2000,6,2430.00 +SO-101108,2025-08-10,West,direct,VE-1000,8,2920.00 +SO-101109,2025-08-10,South,online,VE-1000,7,2555.00 +SO-101110,2025-08-10,North,online,VE-1000,1,365.00 +SO-101111,2025-08-10,South,partner,VE-3000,1,1990.00 +SO-101112,2025-08-10,South,online,VE-1000,9,3120.75 +SO-101113,2025-08-10,South,direct,VE-2000,10,4500.00 +SO-101114,2025-08-10,West,online,VE-1000,3,985.50 +SO-101115,2025-08-10,North,partner,VE-1000,11,4015.00 +SO-101116,2025-08-10,South,online,VE-1000,8,2628.00 +SO-101117,2025-08-10,West,direct,VE-1000,10,3650.00 +SO-101118,2025-08-10,East,direct,VE-1000,14,5110.00 +SO-101119,2025-08-10,South,online,VE-2000,8,3240.00 +SO-101120,2025-08-10,East,online,VE-1000,4,1460.00 +SO-101121,08/10/2025,North,partner,VE-1000,8,2920.00 +SO-101122,2025-08-10,North,direct,VE-2000,8,3600.00 +SO-101123,2025-08-10,East,direct,VE-1000,1,365.00 +SO-101124,2025-08-10,North,direct,VE-2000,14,5670.00 +SO-101125,2025-08-10,East,online,VE-2000,8,3600.00 +SO-101126,2025-08-10,West,direct,VE-2000,7,3150.00 +SO-101127,2025-08-10,East,direct,VE-1000,6,2190.00 +SO-101128,2025-08-11,North,partner,VE-2000,18,8100.00 +SO-101129,2025-08-11,South,partner,VE-1000,6,2190.00 +SO-101130,2025-08-11,East,direct,VE-1000,16,5840.00 +SO-101131,2025-08-11,South,direct,VE-1000,14,5110.00 +SO-101132,2025-08-11,South,online,VE-1000,7,2555.00 +SO-101133,2025-08-11,South,direct,VE-2000,2,855.00 +SO-101134,2025-08-11,West,partner,VE-2000,7,3150.00 +SO-101135,2025-08-11,West,partner,VE-1000,1,365.00 +SO-101136,2025-08-11,West,online,VE-2000,8,3600.00 +SO-101137,2025-08-11,South,online,VE-1000,5,1825.00 +SO-101138,2025-08-11,South,direct,VE-1000,10,3467.50 +SO-101139,2025-08-11,West,partner,VE-2000,5,2137.50 +SO-101140,2025-08-11,West,direct,VE-1000,12,4380.00 +SO-101141,2025-08-11,North,direct,VE-3000,11,20795.50 +SO-101142,2025-08-11,South,direct,VE-2000,23,10350.00 +SO-101143,2025-08-11,North,direct,VE-1000,12,4380.00 +SO-101144,08/11/2025,North,partner,VE-1000,8,2628.00 +SO-101145,2025-08-11,East,direct,VE-3000,3,5373.00 +SO-101146,2025-08-11,East,online,VE-3000,21,41790.00 +SO-101147,2025-08-11,East,partner,VE-2000,8,3240.00 +SO-101148,2025-08-11,East,partner,VE-2000,5,2250.00 +SO-101149,2025-08-12,North,online,VE-1000,3,1040.25 +SO-101150,08/12/2025,North,partner,VE-1000,1,365.00 +SO-101151,2025-08-12,North,online,VE-1000,4,1460.00 +SO-101152,2025-08-12,South,direct,VE-2000,10,4050.00 +SO-101153,2025-08-12,East,partner,VE-3000,3,5373.00 +SO-101154,2025-08-12,South,direct,VE-1000,18,6241.50 +SO-101155,2025-08-12,West,partner,VE-1000,1,346.75 +SO-101156,2025-08-12,East,direct,VE-1000,6,2190.00 +SO-101157,2025-08-12,South,partner,VE-2000,1,405.00 +SO-101158,2025-08-12,North,direct,VE-1000,18,6570.00 +SO-101159,2025-08-12,North,online,VE-1000,4,1387.00 +SO-101160,2025-08-12,North,online,VE-2000,7,2835.00 +SO-101161,2025-08-12,North,direct,VE-1000,3,1040.25 +SO-101162,2025-08-12,North,online,VE-1000,12,3942.00 +SO-101163,2025-08-12,East,direct,VE-1000,4,1314.00 +SO-101164,2025-08-12,South,online,VE-3000,1,1990.00 +SO-101165,2025-08-12,North,partner,VE-3000,8,15920.00 +SO-101166,2025-08-12,North,online,VE-1000,8,2920.00 +SO-101167,2025-08-12,West,partner,VE-2000,9,4050.00 +SO-101168,2025-08-12,North,online,VE-3000,8,15920.00 +SO-101169,2025-08-12,South,online,VE-1000,18,6570.00 +SO-101170,2025-08-12,South,direct,VE-1000,1,365.00 +SO-101171,2025-08-12,North,direct,VE-1000,1,346.75 +SO-101172,2025-08-12,North,direct,VE-2000,6,2430.00 +SO-101173,2025-08-12,East,direct,VE-1000,9,3120.75 +SO-101174,2025-08-12,North,partner,VE-2000,11,4950.00 +SO-101175,2025-08-12,North,partner,VE-3000,8,15920.00 +SO-101176,2025-08-12,East,online,VE-2000,5,2250.00 +SO-101177,2025-08-13,East,direct,VE-2000,2,900.00 +SO-101178,2025-08-13,North,partner,VE-1000,8,2628.00 +SO-101179,2025-08-13,North,partner,VE-2000,11,4950.00 +SO-101180,2025-08-13,North,partner,VE-2000,6,2565.00 +SO-101181,2025-08-13,North,partner,VE-1000,17,6205.00 +SO-101182,2025-08-13,South,direct,VE-2000,21,9450.00 +SO-101183,2025-08-13,East,online,VE-2000,1,450.00 +SO-101184,2025-08-13,East,online,VE-2000,13,5850.00 +SO-101185,2025-08-13,North,direct,VE-2000,5,2025.00 +SO-101186,2025-08-13,West,online,VE-2000,17,7650.00 +SO-101187,2025-08-13,West,online,VE-3000,14,26467.00 +SO-101188,2025-08-13,North,partner,VE-1000,6,2190.00 +SO-101189,2025-08-13,North,direct,VE-2000,12,5400.00 +SO-101190,2025-08-13,East,partner,VE-2000,8,3600.00 +SO-101191,2025-08-13,North,partner,VE-2000,1,450.00 +SO-101192,2025-08-13,North,partner,VE-1000,21,6898.50 +SO-101193,2025-08-13,West,direct,VE-2000,14,5985.00 +SO-101194,2025-08-13,East,partner,VE-1000,12,4380.00 +SO-101195,2025-08-13,South,partner,VE-2000,15,6412.50 +SO-101196,2025-08-13,North,direct,VE-2000,6,2700.00 +SO-101197,2025-08-13,North,direct,VE-1000,4,1460.00 +SO-101198,2025-08-13,North,online,VE-1000,10,3650.00 +SO-101199,2025-08-13,West,partner,VE-1000,12,4380.00 +SO-101200,2025-08-13,West,partner,VE-2000,3,1350.00 +SO-101201,2025-08-13,North,online,VE-3000,9,16119.00 +SO-101202,2025-08-13,East,direct,VE-1000,1,365.00 +SO-101203,2025-08-14,South,online,VE-2000,7,3150.00 +SO-101204,2025-08-14,West,online,VE-2000,11,4950.00 +SO-101205,2025-08-14,East,direct,VE-2000,1,450.00 +SO-101206,2025-08-14,North,online,VE-2000,12,5400.00 +SO-101207,2025-08-14,South,online,VE-1000,1,328.50 +SO-101208,2025-08-14,South,online,VE-3000,5,8955.00 +SO-101209,2025-08-14,North,online,VE-3000,5,9950.00 +SO-101210,08/14/2025,North,online,VE-2000,7,2835.00 +SO-101211,2025-08-14,East,direct,VE-1000,5,1733.75 +SO-101212,2025-08-14,North,partner,VE-1000,1,365.00 +SO-101213,2025-08-14,North,online,VE-1000,1,365.00 +SO-101214,2025-08-14,South,direct,VE-2000,10,4275.00 +SO-101215,2025-08-14,North,online,VE-2000,14,6300.00 +SO-101216,2025-08-14,North,direct,VE-1000,1,365.00 +SO-101217,2025-08-14,North,online,VE-1000,16,5840.00 +SO-101218,2025-08-14,South,partner,VE-2000,6,2700.00 +SO-101219,2025-08-14,South,online,VE-1000,17,5584.50 +SO-101220,2025-08-14,East,direct,VE-1000,1,365.00 +SO-101221,2025-08-14,North,direct,VE-2000,7,3150.00 +SO-101222,2025-08-14,West,direct,VE-1000,11,4015.00 +SO-101223,2025-08-14,South,direct,VE-2000,8,3240.00 +SO-101224,2025-08-14,North,partner,VE-2000,13,5265.00 +SO-101225,2025-08-14,East,partner,VE-3000,1,1990.00 +SO-101226,2025-08-14,South,direct,VE-2000,5,2250.00 +SO-101227,2025-08-14,East,direct,VE-2000,1,405.00 +SO-101228,2025-08-14,East,direct,VE-2000,1,427.50 +SO-101229,2025-08-14,East,direct,VE-1000,8,2920.00 +SO-101230,2025-08-15,South,online,VE-2000,14,5985.00 +SO-101231,2025-08-15,West,online,VE-1000,1,328.50 +SO-101232,2025-08-15,East,direct,VE-1000,16,5840.00 +SO-101233,2025-08-15,East,online,VE-3000,4,7960.00 +SO-101234,2025-08-15,South,direct,VE-1000,1,365.00 +SO-101235,2025-08-15,North,online,VE-1000,19,6935.00 +SO-101236,2025-08-15,South,online,VE-2000,10,4500.00 +SO-101237,2025-08-15,East,online,VE-3000,15,28357.50 +SO-101238,2025-08-15,East,direct,VE-2000,5,2025.00 +SO-101239,2025-08-15,East,online,VE-2000,8,3600.00 +SO-101240,2025-08-15,West,direct,VE-1000,5,1733.75 +SO-101241,2025-08-15,North,partner,VE-1000,9,3285.00 +SO-101242,2025-08-15,West,partner,VE-1000,10,3285.00 +SO-101243,2025-08-15,North,partner,VE-1000,1,365.00 +SO-101244,2025-08-15,South,partner,VE-1000,13,4745.00 +SO-101245,2025-08-15,North,partner,VE-1000,7,2555.00 +SO-101246,2025-08-15,South,online,VE-1000,9,2956.50 +SO-101247,2025-08-15,North,online,VE-2000,15,6075.00 +SO-101248,2025-08-15,West,online,VE-2000,10,4500.00 +SO-101249,2025-08-15,West,online,VE-2000,15,6075.00 +SO-101250,2025-08-15,South,direct,VE-1000,5,1825.00 +SO-101251,2025-08-15,West,direct,VE-1000,7,2299.50 +SO-101252,2025-08-15,North,online,VE-1000,15,5475.00 +SO-101253,2025-08-15,South,partner,VE-1000,1,328.50 +SO-101254,2025-08-15,East,partner,VE-1000,6,2190.00 +SO-101255,2025-08-15,North,direct,VE-1000,1,346.75 +SO-101256,2025-08-15,North,online,VE-1000,7,2299.50 +SO-101257,2025-08-15,East,direct,VE-1000,8,2920.00 +SO-101258,2025-08-15,East,direct,VE-1000,10,3650.00 +SO-101259,2025-08-15,East,direct,VE-1000,12,4380.00 +SO-101260,2025-08-15,South,direct,VE-3000,1,1990.00 +SO-101261,2025-08-16,East,partner,VE-1000,12,4161.00 +SO-101262,2025-08-16,South,partner,VE-2000,11,4950.00 +SO-101263,2025-08-16,East,direct,VE-2000,11,4950.00 +SO-101264,2025-08-16,South,partner,VE-1000,14,4599.00 +SO-101265,2025-08-16,East,partner,VE-2000,2,900.00 +SO-101266,2025-08-16,North,partner,VE-3000,14,27860.00 +SO-101267,2025-08-16,North,online,VE-1000,2,693.50 +SO-101268,2025-08-16,West,direct,VE-1000,5,1733.75 +SO-101269,2025-08-16,South,direct,VE-1000,12,4380.00 +SO-101270,2025-08-16,South,direct,VE-2000,20,9000.00 +SO-101271,2025-08-16,North,partner,VE-2000,12,5400.00 +SO-101272,2025-08-16,East,partner,VE-1000,7,2555.00 +SO-101273,2025-08-16,North,online,VE-2000,6,2700.00 +SO-101274,2025-08-16,East,online,VE-2000,18,8100.00 +SO-101275,2025-08-16,South,online,VE-1000,10,3285.00 +SO-101276,2025-08-16,South,direct,VE-2000,10,4500.00 +SO-101277,2025-08-16,North,online,VE-1000,11,4015.00 +SO-101278,2025-08-16,West,online,VE-3000,3,5970.00 +SO-101279,2025-08-16,North,partner,VE-1000,1,365.00 +SO-101280,2025-08-16,North,online,VE-1000,3,985.50 +SO-101281,2025-08-16,North,partner,VE-1000,17,6205.00 +SO-101282,2025-08-17,West,partner,VE-2000,1,405.00 +SO-101283,2025-08-17,East,direct,VE-1000,3,985.50 +SO-101284,2025-08-17,North,direct,VE-2000,10,4500.00 +SO-101285,2025-08-17,East,direct,VE-1000,19,6935.00 +SO-101286,2025-08-17,East,direct,VE-1000,11,4015.00 +SO-101287,2025-08-17,East,online,VE-2000,9,4050.00 +SO-101288,2025-08-17,East,direct,VE-2000,1,405.00 +SO-101289,2025-08-17,South,online,VE-3000,11,21890.00 +SO-101290,2025-08-17,South,partner,VE-2000,13,5850.00 +SO-101291,2025-08-17,West,online,VE-1000,2,657.00 +SO-101292,2025-08-17,South,online,VE-1000,4,1387.00 +SO-101293,2025-08-17,South,partner,VE-2000,6,2700.00 +SO-101294,2025-08-17,South,direct,VE-2000,5,2137.50 +SO-101295,2025-08-17,East,partner,VE-2000,1,405.00 +SO-101296,2025-08-17,West,online,VE-2000,13,5850.00 +SO-101297,2025-08-17,West,direct,VE-3000,10,17910.00 +SO-101298,2025-08-17,East,online,VE-2000,2,900.00 +SO-101299,2025-08-17,East,direct,VE-2000,1,450.00 +SO-101300,2025-08-17,South,direct,VE-3000,1,1990.00 +SO-101301,2025-08-17,South,direct,VE-3000,1,1990.00 +SO-101302,2025-08-18,West,direct,VE-1000,8,2920.00 +SO-101303,2025-08-18,South,partner,VE-1000,8,2920.00 +SO-101304,2025-08-18,East,partner,VE-2000,7,3150.00 +SO-101305,2025-08-18,South,online,VE-2000,7,3150.00 +SO-101306,2025-08-18,South,online,VE-3000,8,15920.00 +SO-101307,2025-08-18,North,online,VE-1000,9,2956.50 +SO-101308,2025-08-18,North,direct,VE-3000,8,15920.00 +SO-101309,2025-08-18,South,online,VE-1000,7,2299.50 +SO-101310,2025-08-18,West,partner,VE-1000,4,1460.00 +SO-101311,2025-08-18,South,partner,VE-1000,1,365.00 +SO-101312,2025-08-18,East,online,VE-2000,12,5130.00 +SO-101313,2025-08-18,South,partner,VE-3000,5,9950.00 +SO-101314,2025-08-18,North,partner,VE-1000,9,3285.00 +SO-101315,2025-08-18,South,direct,VE-2000,6,2700.00 +SO-101316,2025-08-18,West,partner,VE-1000,1,365.00 +SO-101317,2025-08-18,East,direct,VE-3000,2,3980.00 +SO-101318,2025-08-18,North,direct,VE-1000,7,2555.00 +SO-101319,2025-08-18,South,partner,VE-3000,4,7960.00 +SO-101320,2025-08-18,North,direct,VE-1000,17,6205.00 +SO-101321,2025-08-18,South,online,VE-1000,1,346.75 +SO-101322,2025-08-18,North,direct,VE-3000,11,19701.00 +SO-101323,2025-08-18,East,online,VE-1000,19,6241.50 +SO-101324,2025-08-18,North,online,VE-1000,1,365.00 +SO-101325,2025-08-18,North,partner,VE-2000,16,7200.00 +SO-101326,2025-08-19,East,partner,VE-1000,1,365.00 +SO-101327,2025-08-19,North,partner,VE-1000,1,365.00 +SO-101328,08/19/2025,West,direct,VE-2000,11,4950.00 +SO-101329,2025-08-19,West,direct,VE-1000,19,6241.50 +SO-101330,2025-08-19, South,direct,VE-1000,5,1733.75 +SO-101331,2025-08-19,South,online,VE-1000,10,3467.50 +SO-101332,2025-08-19,North,online,VE-2000,11,4455.00 +SO-101333,2025-08-19,South,direct,VE-2000,10,4500.00 +SO-101334,2025-08-19,West,partner,VE-1000,1,365.00 +SO-101335,2025-08-19,West,partner,VE-1000,10,3650.00 +SO-101336,2025-08-19,West,direct,VE-2000,8,3600.00 +SO-101337,2025-08-19,East,direct,VE-1000,12,4380.00 +SO-101338,2025-08-19,South,partner,VE-1000,10,3467.50 +SO-101339,2025-08-19,West,partner,VE-1000,13,4270.50 +SO-101340,2025-08-19,North,direct,VE-2000,8,3420.00 +SO-101341,2025-08-19,North,direct,VE-1000,9,2956.50 +SO-101342,2025-08-19,South,partner,VE-1000,13,4507.75 +SO-101343,2025-08-19,South,partner,VE-1000,10,3650.00 +SO-101344,08/19/2025,South,direct,VE-1000,5,1642.50 +SO-101345,2025-08-19,East,online,VE-1000,11,4015.00 +SO-101346,2025-08-19,south,partner,VE-3000,6,11940.00 +SO-101347,2025-08-19,South,online,VE-2000,13,5557.50 +SO-101348,2025-08-19,North,partner,VE-1000,6,2080.50 +SO-101349,2025-08-19,North,direct,VE-1000,10,3650.00 +SO-101350,2025-08-19,South,online,VE-1000,14,4599.00 +SO-101351,2025-08-19,East,partner,VE-3000,10,17910.00 +SO-101352,2025-08-19,South,partner,VE-1000,5,1825.00 +SO-101353,2025-08-19,North,online,VE-2000,22,9405.00 +SO-101354,2025-08-19,South,partner,VE-1000,13,4270.50 +SO-101355,2025-08-19,South,online,VE-2000,5,2250.00 +SO-101356,2025-08-19,South,online,VE-3000,9,16119.00 +SO-101357,2025-08-19,North,partner,VE-1000,4,1460.00 +SO-101358,2025-08-19,North,direct,VE-1000,1,328.50 +SO-101359,2025-08-20,South,online,VE-1000,2,693.50 +SO-101360,2025-08-20,East,online,VE-1000,20,7300.00 +SO-101361,2025-08-20,East,partner,VE-2000,12,4860.00 +SO-101362,2025-08-20,East,online,VE-3000,7,13930.00 +SO-101363,2025-08-20,South,online,VE-2000,1,427.50 +SO-101364,2025-08-20,South,partner,VE-3000,3,5373.00 +SO-101365,2025-08-20,East,online,VE-2000,15,6412.50 +SO-101366,2025-08-20,East,partner,VE-1000,18,5913.00 +SO-101367,2025-08-20,West,direct,VE-1000,9,3285.00 +SO-101368,2025-08-20,North,partner,VE-3000,1,1791.00 +SO-101369,2025-08-20,East,online,VE-1000,15,5475.00 +SO-101370,2025-08-20,North,direct,VE-1000,2,657.00 +SO-101371,2025-08-20,North,partner,VE-3000,2,3582.00 +SO-101372,2025-08-20,West,online,VE-1000,3,985.50 +SO-101373,2025-08-20,North,partner,VE-1000,3,1095.00 +SO-101374,2025-08-20,West,direct,VE-1000,13,4745.00 +SO-101375,2025-08-20,South,direct,VE-2000,10,4500.00 +SO-101376,2025-08-20,North,online,VE-1000,9,2956.50 +SO-101377,2025-08-20,West,partner,VE-1000,14,4599.00 +SO-101378,2025-08-20,East,online,VE-1000,15,5475.00 +SO-101379,2025-08-20,North,partner,VE-1000,1,365.00 +SO-101380,2025-08-20,East,online,VE-2000,16,7200.00 +SO-101381,2025-08-20,West,partner,VE-3000,21,41790.00 +SO-101382,2025-08-20,East,online,VE-1000,6,2080.50 +SO-101383,2025-08-20,North,online,VE-1000,7,2427.25 +SO-101384,2025-08-20,North,online,VE-2000,1,427.50 +SO-101385,2025-08-20,South,online,VE-3000,12,23880.00 +SO-101386,2025-08-20,East,direct,VE-2000,9,4050.00 +SO-101387,2025-08-20,East,partner,VE-1000,1,365.00 +SO-101388,2025-08-20,North,partner,VE-3000,1,1890.50 +SO-101389,2025-08-20,North,direct,VE-1000,1,328.50 +SO-101390,2025-08-20,West,online,VE-1000,5,1825.00 +SO-101391,2025-08-20,East,online,VE-1000,13,4745.00 +SO-101392,2025-08-21,South,online,VE-2000,4,1800.00 +SO-101393,2025-08-21,South,direct,VE-1000,20,6935.00 +SO-101394,2025-08-21,South,partner,VE-2000,15,6750.00 +SO-101395,2025-08-21,North,direct,VE-1000,1,365.00 +SO-101396,2025-08-21,North,direct,VE-1000,12,3942.00 +SO-101397,2025-08-21,East,partner,VE-1000,7,2427.25 +SO-101398,2025-08-21,West,partner,VE-1000,6,1971.00 +SO-101399,2025-08-21,East,direct,VE-1000,6,1971.00 +SO-101400,2025-08-21,North,partner,VE-2000,12,5400.00 +SO-101401,2025-08-21,South,partner,VE-2000,5,2250.00 +SO-101402,2025-08-21,North,partner,VE-2000,13,5557.50 +SO-101403,2025-08-21,East,partner,VE-1000,13,4270.50 +SO-101404,2025-08-21,West,partner,VE-3000,11,19701.00 +SO-101405,2025-08-21,East,direct,VE-1000,1,365.00 +SO-101406,2025-08-21,East,direct,VE-2000,9,3847.50 +SO-101407,2025-08-21,West,direct,VE-1000,16,5256.00 +SO-101408,2025-08-21,North,direct,VE-3000,5,9950.00 +SO-101409,2025-08-21,South,online,VE-2000,10,4050.00 +SO-101410,2025-08-21,South,direct,VE-3000,19,37810.00 +SO-101411,2025-08-21,North,partner,VE-2000,8,3600.00 +SO-101412,2025-08-21,South,partner,VE-1000,8,2628.00 +SO-101413,2025-08-21,West,direct,VE-1000,15,5201.25 +SO-101414,2025-08-21,East,direct,VE-1000,1,365.00 +SO-101415,2025-08-21,North,partner,VE-3000,9,17910.00 +SO-101416,2025-08-21,North,online,VE-1000,7,2555.00 +SO-101417,2025-08-21,North,online,VE-2000,7,3150.00 +SO-101418,2025-08-21,West,partner,VE-1000,5,1825.00 +SO-101419,2025-08-22,South,online,VE-2000,15,6750.00 +SO-101420,2025-08-22,West,online,VE-3000,13,23283.00 +SO-101421,2025-08-22,South,online,VE-2000,10,4500.00 +SO-101422,2025-08-22,East,direct,VE-3000,1,1990.00 +SO-101423,2025-08-22,North,direct,VE-3000,1,1990.00 +SO-101424,2025-08-22,West,direct,VE-1000,9,3285.00 +SO-101425,2025-08-22,North,partner,VE-3000,6,11940.00 +SO-101426,2025-08-22,North,direct,VE-1000,9,3285.00 +SO-101427,2025-08-22,West,direct,VE-1000,6,2190.00 +SO-101428,2025-08-22,South,direct,VE-2000,10,4275.00 +SO-101429,2025-08-22,East,direct,VE-3000,17,33830.00 +SO-101430,2025-08-22,South,online,VE-1000,9,2956.50 +SO-101431,2025-08-22,East,partner,VE-3000,17,33830.00 +SO-101432,2025-08-22,East,online,VE-2000,20,8550.00 +SO-101433,2025-08-22,West,direct,VE-2000,4,1800.00 +SO-101434,2025-08-22,East,direct,VE-2000,3,1282.50 +SO-101435,2025-08-22,South,partner,VE-1000,9,3285.00 +SO-101436,2025-08-22,South,online,VE-1000,17,6205.00 +SO-101437,2025-08-22,East,online,VE-2000,3,1215.00 +SO-101438,2025-08-22,West,online,VE-1000,10,3650.00 +SO-101439,2025-08-22,South,direct,VE-1000,5,1733.75 +SO-101440,2025-08-22,North,direct,VE-2000,9,4050.00 +SO-101441,2025-08-22,SOUTH,online,VE-2000,16,6480.00 +SO-101442,2025-08-22,East,online,VE-1000,1,365.00 +SO-101443,2025-08-22,West,direct,VE-1000,9,3285.00 +SO-101444,2025-08-22,East,partner,VE-1000,2,730.00 +SO-101445,2025-08-22,East,direct,VE-1000,8,2774.00 +SO-101446,2025-08-22,South,direct,VE-3000,4,7562.00 +SO-101447,2025-08-22,West,direct,VE-1000,11,4015.00 +SO-101448,2025-08-22,West,online,VE-2000,18,8100.00 +SO-101449,2025-08-23,South,partner,VE-2000,7,3150.00 +SO-101450,2025-08-23,South,online,VE-3000,7,13930.00 +SO-101451,2025-08-23,North,online,VE-3000,3,5970.00 +SO-101452,2025-08-23,East,online,VE-1000,7,2555.00 +SO-101453,2025-08-23,South,direct,VE-2000,11,4455.00 +SO-101454,2025-08-23,South,online,VE-2000,3,1350.00 +SO-101455,2025-08-23,West,online,VE-1000,3,1095.00 +SO-101456,2025-08-23,East,partner,VE-2000,1,450.00 +SO-101457,2025-08-23,West,direct,VE-1000,11,3814.25 +SO-101458,2025-08-23,West,online,VE-2000,4,1800.00 +SO-101459,2025-08-23,North,partner,VE-1000,17,6205.00 +SO-101460,2025-08-23,North,partner,VE-2000,11,4702.50 +SO-101461,2025-08-23,West,direct,VE-3000,29,57710.00 +SO-101462,2025-08-23,South,online,VE-1000,1,346.75 +SO-101463,2025-08-23,North,partner,VE-1000,23,8395.00 +SO-101464,2025-08-23,South,direct,VE-1000,9,3120.75 +SO-101465,2025-08-23,West,online,VE-2000,19,8550.00 +SO-101466,2025-08-24,East,partner,VE-1000,1,365.00 +SO-101467,2025-08-24,North,partner,VE-2000,7,2992.50 +SO-101468,2025-08-24,South,partner,VE-2000,4,1710.00 +SO-101469,2025-08-24,West,partner,VE-1000,9,3120.75 +SO-101470,2025-08-24,South,online,VE-1000,9,3120.75 +SO-101471,2025-08-24,West,online,VE-2000,5,2137.50 +SO-101472,2025-08-24,South,partner,VE-1000,8,2628.00 +SO-101473,2025-08-24,South,partner,VE-2000,13,5557.50 +SO-101474,2025-08-24,West,online,VE-1000,16,5840.00 +SO-101475,2025-08-24,North,online,VE-1000,17,6205.00 +SO-101476,2025-08-24,East,direct,VE-2000,5,2250.00 +SO-101477,2025-08-24,South,partner,VE-1000,4,1387.00 +SO-101478,2025-08-24,West,online,VE-1000,15,4927.50 +SO-101479,2025-08-24,North,direct,VE-2000,3,1350.00 +SO-101480,2025-08-24,East,direct,VE-1000,5,1825.00 +SO-101481,2025-08-24,West,partner,VE-1000,1,346.75 +SO-101482,2025-08-24,East,direct,VE-2000,10,4500.00 +SO-101483,2025-08-24,North,direct,VE-1000,9,3285.00 +SO-101484,2025-08-24,East,direct,VE-3000,17,30447.00 +SO-101485,2025-08-24,North,direct,VE-3000,10,17910.00 +SO-101486,2025-08-24,South,online,VE-1000,4,1387.00 +SO-101487,2025-08-24,East,direct,VE-1000,21,6898.50 +SO-101488,2025-08-24, West,online,VE-1000,6,2190.00 +SO-101489,2025-08-24,North,partner,VE-1000,11,3613.50 +SO-101490,2025-08-24,East,online,VE-1000,10,3650.00 +SO-101491,2025-08-24,North,online,VE-2000,16,6840.00 +SO-101492,2025-08-24,East,online,VE-1000,5,1642.50 +SO-101493,2025-08-24,North,online,VE-2000,11,4455.00 +SO-101494,2025-08-24,West,online,VE-1000,10,3285.00 +SO-101495,2025-08-24,South,partner,VE-2000,22,8910.00 +SO-101496,08/25/2025,South,direct,VE-2000,1,450.00 +SO-101497,2025-08-25,South,online,VE-2000,12,5400.00 +SO-101498,2025-08-25,South,online,VE-2000,10,4050.00 +SO-101499,2025-08-25,West,online,VE-2000,8,3420.00 +SO-101500,2025-08-25,West,direct,VE-2000,1,427.50 +SO-101501,2025-08-25,North,online,VE-2000,9,3645.00 +SO-101502,2025-08-25,North,direct,VE-2000,5,2137.50 +SO-101503,2025-08-25,North,direct,VE-1000,16,5548.00 +SO-101504,2025-08-25,North,direct,VE-2000,9,3847.50 +SO-101505,2025-08-25,East,direct,VE-2000,12,5130.00 +SO-101506,2025-08-25,East,online,VE-1000,6,2190.00 +SO-101507,2025-08-25,West,partner,VE-3000,6,11940.00 +SO-101508,2025-08-25,North,partner,VE-1000,14,5110.00 +SO-101509,2025-08-25,North,online,VE-1000,17,6205.00 +SO-101510,2025-08-25,East,direct,VE-3000,10,18905.00 +SO-101511,2025-08-25,North,online,VE-1000,10,3285.00 +SO-101512,2025-08-25,East,direct,VE-1000,3,1095.00 +SO-101513,2025-08-25,East,direct,VE-2000,13,5850.00 +SO-101514,2025-08-25,South,direct,VE-1000,6,1971.00 +SO-101515,2025-08-25,South,direct,VE-1000,7,2555.00 +SO-101516,2025-08-25,West,direct,VE-3000,9,16119.00 +SO-101517,2025-08-25,North,online,VE-2000,10,4050.00 +SO-101518,2025-08-26,West,direct,VE-2000,8,3600.00 +SO-101519,2025-08-26,North,direct,VE-2000,4,1800.00 +SO-101520,2025-08-26,East,direct,VE-3000,9,17910.00 +SO-101521,2025-08-26,South,partner,VE-1000,16,5840.00 +SO-101522,2025-08-26,West,online,VE-2000,14,5670.00 +SO-101523,2025-08-26,West,online,VE-2000,14,6300.00 +SO-101524,2025-08-26,East,online,VE-1000,3,1095.00 +SO-101525,2025-08-26,North,partner,VE-1000,1,365.00 +SO-101526,2025-08-26,North,partner,VE-1000,5,1642.50 +SO-101527,2025-08-26,East,direct,VE-1000,10,3650.00 +SO-101528,2025-08-26,South,direct,VE-3000,1,1990.00 +SO-101529,2025-08-26,East,partner,VE-2000,7,2835.00 +SO-101530,2025-08-26,East,direct,VE-2000,5,2137.50 +SO-101531,2025-08-26,East,partner,VE-1000,1,365.00 +SO-101532,2025-08-26,South,online,VE-1000,16,5840.00 +SO-101533,2025-08-26,East,direct,VE-1000,12,3942.00 +SO-101534,2025-08-26,West,direct,VE-2000,3,1350.00 +SO-101535,2025-08-26,North,online,VE-1000,12,4380.00 +SO-101536,2025-08-26,North,partner,VE-2000,13,5850.00 +SO-101537,2025-08-27,North,partner,VE-2000,9,4050.00 +SO-101538,2025-08-27,North,partner,VE-1000,5,1825.00 +SO-101539,2025-08-27,North,online,VE-2000,18,7695.00 +SO-101540,2025-08-27,North,online,VE-1000,12,4161.00 +SO-101541,2025-08-27,East,partner,VE-1000,10,3650.00 +SO-101542,2025-08-27,West,partner,VE-1000,13,4745.00 +SO-101543,2025-08-27,East,online,VE-3000,6,11940.00 +SO-101544,2025-08-27,North,direct,VE-2000,8,3600.00 +SO-101545,2025-08-27,North,direct,VE-2000,6,2430.00 +SO-101546,2025-08-27,North,direct,VE-2000,1,427.50 +SO-101547,2025-08-27,East,partner,VE-2000,7,3150.00 +SO-101548,2025-08-27,West,direct,VE-1000,15,5475.00 +SO-101549,2025-08-27,North,direct,VE-2000,25,11250.00 +SO-101550,2025-08-27,South,direct,VE-2000,4,1800.00 +SO-101551,2025-08-27,West,online,VE-1000,1,365.00 +SO-101552,2025-08-27,North,online,VE-2000,7,2992.50 +SO-101553,2025-08-27,West,online,VE-3000,1,1990.00 +SO-101554,2025-08-27,South,direct,VE-3000,5,9950.00 +SO-101555,2025-08-27,East,direct,VE-2000,3,1350.00 +SO-101556,2025-08-27,East,direct,VE-2000,1,450.00 +SO-101557,2025-08-27,West,partner,VE-1000,1,328.50 +SO-101558,2025-08-28,North,online,VE-1000,14,4599.00 +SO-101559,2025-08-28,North,online,VE-1000,8,2628.00 +SO-101560,2025-08-28,North,online,VE-1000,10,3467.50 +SO-101561,2025-08-28,West,online,VE-2000,5,2250.00 +SO-101562,2025-08-28,North,partner,VE-1000,14,5110.00 +SO-101563,2025-08-28,South,online,VE-1000,10,3285.00 +SO-101564,2025-08-28,North,online,VE-2000,8,3420.00 +SO-101565,2025-08-28,South,direct,VE-2000,14,6300.00 +SO-101566,2025-08-28,North,direct,VE-1000,1,346.75 +SO-101567,2025-08-28,South,online,VE-1000,17,6205.00 +SO-101568,2025-08-28,South,partner,VE-2000,1,405.00 +SO-101569,08/28/2025,North,online,VE-3000,6,11343.00 +SO-101570,2025-08-28,North,direct,VE-1000,1,365.00 +SO-101571,2025-08-28,North,direct,VE-2000,20,9000.00 +SO-101572,2025-08-28,West,direct,VE-1000,19,6588.25 +SO-101573,2025-08-28,West,direct,VE-2000,9,3847.50 +SO-101574,2025-08-28,South,online,VE-3000,4,7960.00 +SO-101575,2025-08-28,East,online,VE-2000,3,1215.00 +SO-101576,2025-08-28,South,online,VE-1000,11,3613.50 +SO-101577,2025-08-28,West,online,VE-1000,1,365.00 +SO-101578,2025-08-28,North,partner,VE-2000,1,450.00 +SO-101579,2025-08-29,South,direct,VE-1000,2,730.00 +SO-101580,2025-08-29,East,direct,VE-2000,10,4500.00 +SO-101581,2025-08-29,North,partner,VE-1000,22,8030.00 +SO-101582,2025-08-29,West,direct,VE-1000,7,2427.25 +SO-101583,2025-08-29,South,direct,VE-3000,10,19900.00 +SO-101584,2025-08-29,South,direct,VE-1000,1,328.50 +SO-101585,2025-08-29,East,partner,VE-2000,8,3600.00 +SO-101586,2025-08-29,West,online,VE-2000,21,9450.00 +SO-101587,2025-08-29,West,online,VE-1000,1,328.50 +SO-101588,2025-08-29,South,online,VE-1000,15,5475.00 +SO-101589,2025-08-29,West,direct,VE-2000,15,6750.00 +SO-101590,2025-08-29,East,partner,VE-3000,15,28357.50 +SO-101591,2025-08-29,North,online,VE-1000,1,328.50 +SO-101592,2025-08-29,South,partner,VE-3000,8,15920.00 +SO-101593,2025-08-29,North,partner,VE-2000,14,6300.00 +SO-101594,2025-08-29,South,online,VE-2000,4,1710.00 +SO-101595,2025-08-29,South,direct,VE-1000,5,1733.75 +SO-101596,2025-08-29,North,partner,VE-3000,5,9452.50 +SO-101597,2025-08-29,East,direct,VE-1000,2,730.00 +SO-101598,2025-08-29,West,online,VE-1000,9,3285.00 +SO-101599,2025-08-29,West,partner,VE-1000,19,6935.00 +SO-101600,2025-08-29,North,partner,VE-1000,5,1825.00 +SO-101601,2025-08-29,North,partner,VE-2000,1,450.00 +SO-101602,2025-08-29,South,direct,VE-2000,1,450.00 +SO-101603,2025-08-30,South,online,VE-2000,3,1282.50 +SO-101604,2025-08-30,South,direct,VE-3000,11,20795.50 +SO-101605,2025-08-30,South,online,VE-2000,19,8550.00 +SO-101606,2025-08-30,South,online,VE-2000,1,450.00 +SO-101607,2025-08-30,West,partner,VE-2000,14,6300.00 +SO-101608,2025-08-30,South,partner,VE-2000,8,3420.00 +SO-101609,2025-08-30,East,online,VE-1000,6,2190.00 +SO-101610,2025-08-30,North,online,VE-2000,1,427.50 +SO-101611,2025-08-30,South,partner,VE-3000,7,13930.00 +SO-101612,2025-08-30,North,partner,VE-2000,8,3600.00 +SO-101613,2025-08-30,North,direct,VE-1000,1,365.00 +SO-101614,2025-08-30,West,partner,VE-1000,13,4745.00 +SO-101615,2025-08-30,East,online,VE-1000,13,4745.00 +SO-101616,2025-08-30,South,partner,VE-1000,3,985.50 +SO-101617,2025-08-30,North,direct,VE-2000,7,3150.00 +SO-101618,2025-08-30,North,direct,VE-3000,10,19900.00 +SO-101619,2025-08-30,South,direct,VE-2000,6,2430.00 +SO-101620,2025-08-30,North,online,VE-3000,4,7960.00 +SO-101621,2025-08-30,North,online,VE-1000,10,3650.00 +SO-101622,2025-08-30,North,direct,VE-1000,12,4380.00 +SO-101623,2025-08-30,East,online,VE-2000,10,4500.00 +SO-101624,2025-08-30,East,partner,VE-3000,10,18905.00 +SO-101625,2025-08-30,North,partner,VE-3000,7,12537.00 +SO-101626,2025-08-30,West,partner,VE-1000,4,1460.00 +SO-101627,2025-08-30,South,direct,VE-3000,7,13930.00 +SO-101628,2025-08-30,South,direct,VE-1000,16,5840.00 +SO-101629,2025-08-30,South,partner,VE-2000,3,1350.00 +SO-101630,2025-08-30,West,partner,VE-1000,8,2920.00 +SO-101631,2025-08-30,East,online,VE-1000,16,5840.00 +SO-101632,2025-08-30,East,online,VE-1000,9,3285.00 +SO-101633,2025-08-31,South,partner,VE-1000,12,4380.00 +SO-101634,2025-08-31,South,direct,VE-3000,5,9950.00 +SO-101635,2025-08-31,East,online,VE-1000,16,5840.00 +SO-101636,2025-08-31,East,direct,VE-1000,2,657.00 +SO-101637,2025-08-31,South,partner,VE-1000,15,5475.00 +SO-101638,2025-08-31,East,online,VE-1000,9,3285.00 +SO-101639,2025-08-31,North,direct,VE-1000,8,2920.00 +SO-101640,2025-08-31,West,direct,VE-2000,19,7695.00 +SO-101641,2025-08-31,East,partner,VE-2000,10,4050.00 +SO-101642,2025-08-31,North,online,VE-1000,12,3942.00 +SO-101643,2025-08-31,East,online,VE-2000,13,5557.50 +SO-101644,2025-08-31,West,online,VE-1000,8,2920.00 +SO-101645,2025-08-31,South,direct,VE-1000,9,3120.75 +SO-101646,2025-08-31,North,direct,VE-2000,10,4050.00 +SO-101647,2025-08-31,West,partner,VE-2000,4,1710.00 +SO-101648,2025-08-31,North,online,VE-1000,1,365.00 +SO-101649,2025-08-31,North,partner,VE-2000,1,450.00 +SO-101650,2025-08-31,North,direct,VE-1000,1,365.00 +SO-101651,2025-08-31,South,partner,VE-1000,6,1971.00 +SO-101652,2025-08-31,North,direct,VE-3000,16,31840.00 +SO-101653,2025-08-31,North,online,VE-3000,6,11940.00 +SO-101654,2025-08-31,South,direct,VE-1000,5,1642.50 +SO-101655,2025-08-31,East,partner,VE-2000,11,4950.00 +SO-101656,2025-08-31,South,online,VE-1000,8,2920.00 +SO-101657,2025-08-31,South,online,VE-2000,3,1215.00 +SO-101658,2025-08-31,West,online,VE-2000,6,2700.00 +SO-101659,2025-08-31,North,partner,VE-1000,2,693.50 +SO-101660,2025-09-01,South,online,VE-1000,1,365.00 +SO-101661,2025-09-01,North,partner,VE-1000,10,3650.00 +SO-101662,2025-09-01,North,online,VE-3000,14,25074.00 +SO-101663,2025-09-01,South,direct,VE-2000,1,427.50 +SO-101664,2025-09-01,East,direct,VE-1000,4,1460.00 +SO-101665,09/01/2025,South,online,VE-1000,11,3814.25 +SO-101666,2025-09-01,West,direct,VE-2000,19,8550.00 +SO-101667,2025-09-01,North,direct,VE-2000,9,4050.00 +SO-101668,2025-09-01,North,online,VE-1000,14,5110.00 +SO-101669,2025-09-01,East,partner,VE-1000,1,365.00 +SO-101670,2025-09-01,North,online,VE-2000,8,3420.00 +SO-101671,2025-09-01,West,online,VE-1000,7,2299.50 +SO-101672,2025-09-01,North,partner,VE-1000,3,1095.00 +SO-101673,2025-09-01,South,partner,VE-1000,12,4380.00 +SO-101674,2025-09-01,East,direct,VE-1000,2,730.00 +SO-101675,2025-09-01,South,partner,VE-1000,13,4745.00 +SO-101676,2025-09-01,North,partner,VE-3000,6,10746.00 +SO-101677,2025-09-01,South,direct,VE-2000,8,3240.00 +SO-101678,2025-09-01,North,direct,VE-1000,23,8395.00 +SO-101679,2025-09-01,East,direct,VE-1000,14,4854.50 +SO-101680,2025-09-01,North,partner,VE-2000,1,450.00 +SO-101681,2025-09-01,North,partner,VE-2000,17,7267.50 +SO-101682,2025-09-01,East,direct,VE-3000,6,11940.00 +SO-101683,2025-09-01,West,online,VE-1000,6,2190.00 +SO-101684,2025-09-01,West,online,VE-1000,1,365.00 +SO-101685,2025-09-01,South,partner,VE-3000,13,25870.00 +SO-101686,2025-09-01,West,direct,VE-3000,4,7960.00 +SO-101687,2025-09-01,West,direct,VE-1000,10,3285.00 +SO-101688,2025-09-01,East,direct,VE-1000,11,4015.00 +SO-101689,2025-09-01,South,direct,VE-1000,12,3942.00 +SO-101690,2025-09-01,South,direct,VE-1000,4,1460.00 +SO-101691,2025-09-01,North,online,VE-1000,10,3650.00 +SO-101692,2025-09-02,East,partner,VE-2000,14,5670.00 +SO-101693,2025-09-02,East,direct,VE-2000,6,2430.00 +SO-101694,2025-09-02,East,direct,VE-1000,16,5840.00 +SO-101695,2025-09-02,West,partner,VE-3000,1,1890.50 +SO-101696,2025-09-02,North,online,VE-1000,1,365.00 +SO-101697,2025-09-02,North,online,VE-1000,11,4015.00 +SO-101698,2025-09-02,South,online,VE-2000,10,4500.00 +SO-101699,2025-09-02,East,direct,VE-2000,12,4860.00 +SO-101700,2025-09-02,South,partner,VE-1000,14,5110.00 +SO-101701,2025-09-02,West,partner,VE-2000,10,4500.00 +SO-101702,2025-09-02,North,direct,VE-1000,10,3467.50 +SO-101703,2025-09-02,West,online,VE-3000,8,14328.00 +SO-101704,2025-09-02,East,partner,VE-3000,6,11940.00 +SO-101705,2025-09-02,North,partner,VE-1000,1,365.00 +SO-101706,2025-09-02,South,partner,VE-1000,10,3650.00 +SO-101707,2025-09-02,East,partner,VE-1000,9,3285.00 +SO-101708,2025-09-02,East,direct,VE-2000,7,3150.00 +SO-101709,2025-09-02,North,partner,VE-3000,2,3980.00 +SO-101710,2025-09-02,West,online,VE-2000,8,3600.00 +SO-101711,2025-09-02,East,direct,VE-1000,5,1825.00 +SO-101712,2025-09-02,North,direct,VE-3000,8,14328.00 +SO-101713,2025-09-02,East,online,VE-1000,11,3814.25 +SO-101714,2025-09-03,North,online,VE-1000,13,4507.75 +SO-101715,2025-09-03,East,direct,VE-2000,9,4050.00 +SO-101716,2025-09-03,North,online,VE-1000,6,2080.50 +SO-101717,2025-09-03,East,partner,VE-3000,12,22686.00 +SO-101718,2025-09-03,North,direct,VE-1000,10,3650.00 +SO-101719,2025-09-03,West,online,VE-1000,1,365.00 +SO-101720,2025-09-03,East,online,VE-1000,12,4380.00 +SO-101721,2025-09-03,West,online,VE-2000,13,5557.50 +SO-101722,2025-09-03,South,online,VE-2000,17,7267.50 +SO-101723,2025-09-03,North,direct,VE-1000,12,3942.00 +SO-101724,2025-09-03,South,online,VE-1000,11,4015.00 +SO-101725,2025-09-03,North,partner,VE-2000,10,4500.00 +SO-101726,2025-09-03,East,partner,VE-1000,6,2080.50 +SO-101727,2025-09-03,North,direct,VE-1000,16,5840.00 +SO-101728,2025-09-03,South,direct,VE-2000,22,8910.00 +SO-101729,2025-09-03,South,direct,VE-1000,9,2956.50 +SO-101730,2025-09-03,North,partner,VE-2000,2,810.00 +SO-101731,2025-09-03,East,direct,VE-2000,1,427.50 +SO-101732,2025-09-03,West,online,VE-2000,2,855.00 +SO-101733,2025-09-03,South,partner,VE-2000,8,3420.00 +SO-101734,2025-09-04,North,online,VE-2000,5,2250.00 +SO-101735,2025-09-04,East,online,VE-3000,5,8955.00 +SO-101736,2025-09-04,North,direct,VE-2000,14,5670.00 +SO-101737,2025-09-04,North,direct,VE-1000,11,3613.50 +SO-101738,2025-09-04,West,partner,VE-2000,10,4500.00 +SO-101739,09/04/2025,North,partner,VE-2000,8,3600.00 +SO-101740,2025-09-04,North,online,VE-1000,6,2190.00 +SO-101741,2025-09-04,East,partner,VE-1000,11,4015.00 +SO-101742,2025-09-04,North,partner,VE-2000,14,6300.00 +SO-101743,2025-09-04,North,direct,VE-3000,11,21890.00 +SO-101744,2025-09-04,East,online,VE-2000,8,3600.00 +SO-101745,2025-09-04,South,direct,VE-1000,14,4854.50 +SO-101746,2025-09-04,West,online,VE-3000,5,9452.50 +SO-101747,2025-09-04,South,direct,VE-2000,5,2025.00 +SO-101748,2025-09-04,West,partner,VE-1000,1,365.00 +SO-101749,2025-09-04,South,direct,VE-2000,9,4050.00 +SO-101750,2025-09-04,East,direct,VE-1000,5,1825.00 +SO-101751,2025-09-04,East,direct,VE-1000,2,730.00 +SO-101752,2025-09-04,West,direct,VE-2000,10,4500.00 +SO-101753,2025-09-04,West,partner,VE-1000,18,5913.00 +SO-101754,2025-09-04,North,online,VE-2000,17,7267.50 +SO-101755,2025-09-04,North,direct,VE-2000,3,1350.00 +SO-101756,2025-09-04,East,partner,VE-1000,8,2774.00 +SO-101757,2025-09-04,North,direct,VE-1000,9,3120.75 +SO-101758,2025-09-05,North,direct,VE-1000,7,2299.50 +SO-101759,2025-09-05,West,partner,VE-2000,12,4860.00 +SO-101760,2025-09-05,South,direct,VE-2000,10,4050.00 +SO-101761,2025-09-05,North,online,VE-1000,2,730.00 +SO-101762,2025-09-05,South,direct,VE-2000,13,5850.00 +SO-101763,2025-09-05,South,online,VE-3000,14,26467.00 +SO-101764,2025-09-05,East,online,VE-1000,4,1314.00 +SO-101765,2025-09-05,North,online,VE-2000,7,3150.00 +SO-101766,2025-09-05,South,direct,VE-1000,1,365.00 +SO-101767,2025-09-05,East,online,VE-2000,14,5985.00 +SO-101768,2025-09-05,North,partner,VE-1000,7,2555.00 +SO-101769,2025-09-05,East,direct,VE-2000,1,427.50 +SO-101770,2025-09-05,South,online,VE-3000,1,1890.50 +SO-101771,2025-09-05,North,online,VE-2000,9,3645.00 +SO-101772,2025-09-05,North,partner,VE-1000,10,3467.50 +SO-101773,2025-09-05,West,direct,VE-2000,14,6300.00 +SO-101774,2025-09-05,East,partner,VE-1000,5,1825.00 +SO-101775,2025-09-05,West,direct,VE-1000,1,328.50 +SO-101776,2025-09-05,West,partner,VE-1000,4,1460.00 +SO-101777,2025-09-05,North,partner,VE-1000,20,6935.00 +SO-101778,2025-09-05,East,direct,VE-1000,19,6588.25 +SO-101779,2025-09-05,South,online,VE-2000,5,2250.00 +SO-101780,2025-09-05,East,direct,VE-2000,2,900.00 +SO-101781,2025-09-05,West,partner,VE-1000,1,365.00 +SO-101782,2025-09-05,West,partner,VE-3000,11,19701.00 +SO-101783,2025-09-05,North,online,VE-1000,14,4854.50 +SO-101784,2025-09-05,East,online,VE-1000,9,3285.00 +SO-101785,2025-09-05,North,online,VE-2000,8,3600.00 +SO-101786,2025-09-05,West,direct,VE-2000,10,4500.00 +SO-101787,2025-09-05,North,online,VE-3000,1,1990.00 +SO-101788,2025-09-05,East,partner,VE-2000,10,4275.00 +SO-101789,2025-09-05,South,online,VE-1000,16,5840.00 +SO-101790,2025-09-05,East,online,VE-2000,6,2430.00 +SO-101791,2025-09-05,East,direct,VE-3000,1,1791.00 +SO-101792,2025-09-06,South,partner,VE-2000,10,4275.00 +SO-101793,2025-09-06,West,direct,VE-1000,7,2555.00 +SO-101794,2025-09-06,East,direct,VE-2000,10,4500.00 +SO-101795,2025-09-06,North,online,VE-1000,11,3613.50 +SO-101796,2025-09-06,North,online,VE-1000,3,1095.00 +SO-101797,2025-09-06,North,direct,VE-3000,12,23880.00 +SO-101798,2025-09-06,East,direct,VE-3000,8,15124.00 +SO-101799,2025-09-06,West,direct,VE-1000,17,6205.00 +SO-101800,2025-09-06,North,partner,VE-2000,4,1620.00 +SO-101801,2025-09-06,South,online,VE-3000,10,19900.00 +SO-101802,2025-09-06,South,online,VE-1000,8,2628.00 +SO-101803,2025-09-06,South,partner,VE-2000,9,4050.00 +SO-101804,2025-09-06,West,partner,VE-1000,10,3650.00 +SO-101805,2025-09-06,North,direct,VE-2000,19,8122.50 +SO-101806,2025-09-06,West,partner,VE-1000,18,6570.00 +SO-101807,2025-09-06,West,online,VE-3000,1,1890.50 +SO-101808,2025-09-06,North,direct,VE-1000,1,365.00 +SO-101809,2025-09-06,North,online,VE-1000,8,2920.00 +SO-101810,2025-09-06,East,direct,VE-2000,13,5850.00 +SO-101811,2025-09-06,West,online,VE-2000,13,5557.50 +SO-101812,2025-09-06,East,direct,VE-2000,21,8977.50 +SO-101813,2025-09-06,South,partner,VE-1000,5,1825.00 +SO-101814,2025-09-06,East,partner,VE-2000,10,4050.00 +SO-101815,2025-09-06,South,direct,VE-1000,11,4015.00 +SO-101816,2025-09-06,South,partner,VE-1000,1,365.00 +SO-101817,2025-09-06,North,online,VE-1000,5,1825.00 +SO-101818,2025-09-06,North,partner,VE-1000,14,5110.00 +SO-101819,2025-09-07,South,partner,VE-2000,13,5850.00 +SO-101820,2025-09-07,South,direct,VE-1000,24,8760.00 +SO-101821,2025-09-07,North,online,VE-1000,16,5840.00 +SO-101822,2025-09-07,North,partner,VE-2000,9,4050.00 +SO-101823,2025-09-07,North,online,VE-3000,13,23283.00 +SO-101824,2025-09-07,East,online,VE-1000,15,5475.00 +SO-101825,2025-09-07,East,direct,VE-2000,7,2992.50 +SO-101826,2025-09-07,East,online,VE-1000,10,3467.50 +SO-101827,2025-09-07,East,online,VE-1000,5,1642.50 +SO-101828,2025-09-07,West,partner,VE-1000,7,2555.00 +SO-101829,2025-09-07,South,direct,VE-2000,4,1800.00 +SO-101830,2025-09-07,West,direct,VE-2000,25,10125.00 +SO-101831,2025-09-07,North,online,VE-2000,16,7200.00 +SO-101832,2025-09-07,South,online,VE-2000,9,4050.00 +SO-101833,2025-09-07,North,partner,VE-1000,18,6570.00 +SO-101834,2025-09-07,West,online,VE-2000,7,3150.00 +SO-101835,2025-09-07,West,online,VE-3000,11,19701.00 +SO-101836,2025-09-07,West,partner,VE-2000,15,6075.00 +SO-101837,2025-09-07,East,partner,VE-1000,9,3285.00 +SO-101838,2025-09-07,East,partner,VE-1000,9,3285.00 +SO-101839,2025-09-07,West,online,VE-1000,1,328.50 +SO-101840,2025-09-07,North,direct,VE-1000,1,328.50 +SO-101841,2025-09-07,North,online,VE-2000,3,1350.00 +SO-101842,2025-09-07,East,partner,VE-1000,16,5840.00 +SO-101843,2025-09-08,East,direct,VE-2000,10, +SO-101844,09/08/2025,West,direct,VE-2000,7,3150.00 +SO-101845,2025-09-08,South,partner,VE-2000,14,6300.00 +SO-101846,2025-09-08,West,direct,VE-2000,1,450.00 +SO-101847,2025-09-08,West,online,VE-3000,12,23880.00 +SO-101848,2025-09-08,West,direct,VE-1000,10,3650.00 +SO-101849,2025-09-08,North,direct,VE-2000,7,3150.00 +SO-101850,2025-09-08,South,direct,VE-1000,15,5475.00 +SO-101851,2025-09-08,West,direct,VE-1000,12,3942.00 +SO-101852,2025-09-08,North,direct,VE-1000,14,4854.50 +SO-101853,2025-09-08,South,partner,VE-1000,3,1040.25 +SO-101854,2025-09-08,South,online,VE-3000,11,21890.00 +SO-101855,2025-09-08,North,direct,VE-1000,14,5110.00 +SO-101856,2025-09-08,North,online,VE-1000,4,1387.00 +SO-101857,2025-09-08,West,direct,VE-2000,14,6300.00 +SO-101858,2025-09-08,South,partner,VE-2000,16,7200.00 +SO-101859,2025-09-08,North,online,VE-2000,3,1350.00 +SO-101860,2025-09-08,South,direct,VE-1000,9,3285.00 +SO-101861,2025-09-08,North,direct,VE-1000,1,346.75 +SO-101862,2025-09-08,East,partner,VE-2000,3,1215.00 +SO-101863,2025-09-08,West,direct,VE-2000,2,855.00 +SO-101864,2025-09-08,West,partner,VE-2000,4,1800.00 +SO-101865,2025-09-08,South,partner,VE-1000,10,3650.00 +SO-101866,2025-09-08,North,online,VE-1000,4,1460.00 +SO-101867,2025-09-08,South,online,VE-1000,2,730.00 +SO-101868,2025-09-09,East,online,VE-2000,14,6300.00 +SO-101869,2025-09-09,North,direct,VE-3000,10,19900.00 +SO-101870,2025-09-09,South,partner,VE-2000,4,1800.00 +SO-101871,2025-09-09,north,partner,VE-2000,8,3420.00 +SO-101872,2025-09-09,North,direct,VE-1000,3,1095.00 +SO-101873,2025-09-09,South,online,VE-1000,2,657.00 +SO-101874,2025-09-09,North,direct,VE-2000,22,9900.00 +SO-101875,2025-09-09,West,online,VE-1000,6,2190.00 +SO-101876,2025-09-09,South,partner,VE-2000,10,4500.00 +SO-101877,2025-09-09,West,direct,VE-1000,13,4745.00 +SO-101878,2025-09-09,North,partner,VE-1000,10,3285.00 +SO-101879,2025-09-09,South,direct,VE-2000,10,4500.00 +SO-101880,2025-09-09,West,partner,VE-1000,13,4745.00 +SO-101881,2025-09-09,South,online,VE-1000,9,2956.50 +SO-101882,2025-09-09,North,online,VE-1000,8,2628.00 +SO-101883,2025-09-09,West,partner,VE-2000,14,5985.00 +SO-101884,2025-09-09,South,direct,VE-1000,9,3285.00 +SO-101885,2025-09-09,West,online,VE-1000,8,2774.00 +SO-101886,2025-09-09,West,online,VE-2000,18,7695.00 +SO-101887,2025-09-09,East,online,VE-2000,8,3600.00 +SO-101888,2025-09-09,West,online,VE-3000,1,1990.00 +SO-101889,2025-09-09,North,online,VE-1000,4,1460.00 +SO-101890,2025-09-09,South,online,VE-2000,13,5557.50 +SO-101891,2025-09-09,East,online,VE-2000,1,450.00 +SO-101892,2025-09-10,East,online,VE-1000,4,1460.00 +SO-101893,2025-09-10,West,partner,VE-1000,1,328.50 +SO-101894,2025-09-10,East,partner,VE-1000,11,4015.00 +SO-101895,2025-09-10,West,partner,VE-1000,10,3650.00 +SO-101896,2025-09-10,East,direct,VE-2000,6,2700.00 +SO-101897,2025-09-10,North,direct,VE-2000,16,6480.00 +SO-101898,2025-09-10,South,direct,VE-1000,12,3942.00 +SO-101899,2025-09-10,East,online,VE-2000,2,900.00 +SO-101900,2025-09-10,South,partner,VE-2000,16,7200.00 +SO-101901,2025-09-10,North,direct,VE-1000,20,6935.00 +SO-101902,2025-09-10,North,online,VE-3000,8,15124.00 +SO-101903,2025-09-10,North,partner,VE-3000,10,18905.00 +SO-101904,2025-09-10,South,direct,VE-1000,1,346.75 +SO-101905,2025-09-10,West,online,VE-1000,11,3814.25 +SO-101906,2025-09-10,East,direct,VE-1000,9,3285.00 +SO-101907,2025-09-10,South,online,VE-2000,11,4950.00 +SO-101908,2025-09-10,West,direct,VE-1000,5,1825.00 +SO-101909,2025-09-10,South,direct,VE-2000,12,4860.00 +SO-101910,2025-09-10,East,direct,VE-1000,4,1314.00 +SO-101911,2025-09-10,South,direct,VE-3000,4,7562.00 +SO-101912,2025-09-10,North,partner,VE-3000,13,24576.50 +SO-101913,2025-09-10,East,direct,VE-1000,10,3650.00 +SO-101914,2025-09-10,South,online,VE-3000,16,28656.00 +SO-101915,2025-09-11,North,online,VE-3000,5,8955.00 +SO-101916,2025-09-11,East,partner,VE-2000,15,6750.00 +SO-101917,2025-09-11,North,direct,VE-1000,1,365.00 +SO-101918,2025-09-11,South,direct,VE-1000,7,2555.00 +SO-101919,2025-09-11,North,partner,VE-2000,7,2992.50 +SO-101920,2025-09-11,South,online,VE-2000,13,5265.00 +SO-101921,2025-09-11,North,partner,VE-1000,2,730.00 +SO-101922,2025-09-11,West,partner,VE-1000,8,2628.00 +SO-101923,2025-09-11,East,direct,VE-2000,16,7200.00 +SO-101924,2025-09-11,North,partner,VE-2000,5,2250.00 +SO-101925,2025-09-11,East,partner,VE-1000,8,2920.00 +SO-101926,2025-09-11,South,partner,VE-3000,4,7960.00 +SO-101927,2025-09-11,East,direct,VE-3000,13,25870.00 +SO-101928,2025-09-11,East,partner,VE-1000,4,1314.00 +SO-101929,2025-09-11,North,partner,VE-3000,8,14328.00 +SO-101930,2025-09-11,North,partner,VE-1000,8,2774.00 +SO-101931,2025-09-11,East,online,VE-1000,13,4745.00 +SO-101932,2025-09-11,North,partner,VE-2000,3,1350.00 +SO-101933,2025-09-11,West,direct,VE-2000,1,427.50 +SO-101934,2025-09-11,South,partner,VE-1000,6,1971.00 +SO-101935,2025-09-11,North,online,VE-1000,9,3285.00 +SO-101936,2025-09-11,West,online,VE-1000,11,3613.50 +SO-101937,2025-09-11,East,online,VE-1000,6,2080.50 +SO-101938,2025-09-11,East,online,VE-3000,9,17910.00 +SO-101939,2025-09-11,South,direct,VE-3000,16,30248.00 +SO-101940,2025-09-12,East,direct,VE-1000,8,2920.00 +SO-101941,2025-09-12,East,partner,VE-3000,18,35820.00 +SO-101942,2025-09-12,West,online,VE-1000,1,346.75 +SO-101943,2025-09-12,North,online,VE-2000,3,1350.00 +SO-101944,2025-09-12,East,direct,VE-1000,9,3285.00 +SO-101945,2025-09-12,South,direct,VE-3000,16,30248.00 +SO-101946,2025-09-12,South,direct,VE-1000,10,3467.50 +SO-101947,2025-09-12,North,partner,VE-3000,1,1990.00 +SO-101948,2025-09-12,South,partner,VE-3000,3,5970.00 +SO-101949,2025-09-12,North,online,VE-2000,7,3150.00 +SO-101950,2025-09-12,North,online,VE-1000,3,985.50 +SO-101951,2025-09-12,East,direct,VE-1000,9,3285.00 +SO-101952,2025-09-12,South,partner,VE-1000,5,1825.00 +SO-101953,2025-09-12,East,online,VE-2000,14,6300.00 +SO-101954,2025-09-13,North,partner,VE-1000,5,1825.00 +SO-101955,2025-09-13,South,partner,VE-1000,7,2555.00 +SO-101956,2025-09-13,East,direct,VE-2000,7,3150.00 +SO-101957,2025-09-13,North,online,VE-1000,11,4015.00 +SO-101958,2025-09-13,South,online,VE-1000,15,5475.00 +SO-101959,2025-09-13,East,online,VE-1000,9,3285.00 +SO-101960,2025-09-13,East,direct,VE-2000,7,2992.50 +SO-101961,2025-09-13,North,online,VE-3000,2,3980.00 +SO-101962,2025-09-13,West,direct,VE-2000,6,2700.00 +SO-101963,09/13/2025,North,online,VE-1000,5,1733.75 +SO-101964,2025-09-13,South,direct,VE-1000,9,3285.00 +SO-101965,2025-09-13,North,online,VE-2000,11,4455.00 +SO-101966,2025-09-13,South,direct,VE-1000,4,1460.00 +SO-101967,2025-09-13,North,partner,VE-3000,1,1791.00 +SO-101968,2025-09-13,East,partner,VE-1000,20,7300.00 +SO-101969,2025-09-13,West,online,VE-2000,3,1282.50 +SO-101970,2025-09-13,North,online,VE-1000,3,985.50 +SO-101971,2025-09-13,West,online,VE-2000,18,7290.00 +SO-101972,2025-09-13,South,online,VE-1000,16,5840.00 +SO-101973,2025-09-13,North,online,VE-1000,12,4380.00 +SO-101974,2025-09-13,WEST,direct,VE-2000,1,405.00 +SO-101975,2025-09-14,West,partner,VE-3000,6,11343.00 +SO-101976,2025-09-14,East,partner,VE-1000,12,4161.00 +SO-101977,2025-09-14, East,direct,VE-1000,5,1825.00 +SO-101978,2025-09-14,South,direct,VE-2000,1,450.00 +SO-101979,2025-09-14,East,partner,VE-1000,18,6241.50 +SO-101980,2025-09-14,North,online,VE-2000,15,6075.00 +SO-101981,2025-09-14,West,online,VE-1000,7,2555.00 +SO-101982,2025-09-14,East,partner,VE-1000,7,2427.25 +SO-101983,2025-09-14,East,online,VE-3000,9,17014.50 +SO-101984,2025-09-14,South,direct,VE-3000,1,1990.00 +SO-101985,2025-09-14,West,direct,VE-1000,5,1642.50 +SO-101986,2025-09-14,North,partner,VE-1000,6,2190.00 +SO-101987,2025-09-14,North,direct,VE-2000,8,3600.00 +SO-101988,2025-09-14,South,partner,VE-2000,4,1800.00 +SO-101989,2025-09-14,South,online,VE-1000,3,985.50 +SO-101990,2025-09-14,North,online,VE-1000,1,365.00 +SO-101991,2025-09-14,East,online,VE-2000,11,4950.00 +SO-101992,2025-09-14,West,online,VE-3000,13,25870.00 +SO-101993,2025-09-14,North,direct,VE-2000,15,6075.00 +SO-101994,2025-09-14,West,online,VE-1000,10,3650.00 +SO-101995,2025-09-14,North,direct,VE-1000,9,3285.00 +SO-101996,2025-09-14,East,online,VE-2000,3,1350.00 +SO-101997,2025-09-14,East,direct,VE-1000,13,4507.75 +SO-101998,2025-09-14,West,online,VE-2000,16,7200.00 +SO-101999,2025-09-15,South,online,VE-1000,8,2774.00 +SO-102000,2025-09-15,East,partner,VE-2000,9,4050.00 +SO-102001,2025-09-15,West,direct,VE-1000,1,346.75 +SO-102002,2025-09-15,East,online,VE-1000,15,5475.00 +SO-102003,2025-09-15,South,online,VE-2000,14,5985.00 +SO-102004,2025-09-15,east,online,VE-1000,13,4745.00 +SO-102005,2025-09-15,North,direct,VE-2000,11,4702.50 +SO-102006,2025-09-15,East,partner,VE-1000,1,346.75 +SO-102007,2025-09-15,South,direct,VE-2000,16,6480.00 +SO-102008,2025-09-15,East,direct,VE-3000,10,17910.00 +SO-102009,2025-09-15,East,online,VE-1000,2,730.00 +SO-102010,2025-09-15,South,partner,VE-1000,9,3285.00 +SO-102011,2025-09-15,East,direct,VE-1000,7,2555.00 +SO-102012,2025-09-15,West,direct,VE-2000,14,6300.00 +SO-102013,2025-09-15,North,direct,VE-2000,15,6750.00 +SO-102014,2025-09-15,North,online,VE-1000,10,3467.50 +SO-102015,2025-09-15,North,direct,VE-1000,12,3942.00 +SO-102016,2025-09-15,North,online,VE-2000,9,3645.00 +SO-102017,2025-09-15,South,online,VE-1000,6,2190.00 +SO-102018,2025-09-15,East,direct,VE-1000,1,365.00 +SO-102019,2025-09-15,North,online,VE-2000,9,4050.00 +SO-102020,2025-09-15,North,direct,VE-3000,7,13930.00 +SO-102021,2025-09-15,North,partner,VE-2000,7,2835.00 +SO-102022,2025-09-15,East,direct,VE-3000,2,3980.00 +SO-102023,2025-09-15,South,partner,VE-1000,6,2190.00 +SO-102024,2025-09-16,West,partner,VE-2000,7,3150.00 +SO-102025,2025-09-16,West,online,VE-1000,12,4380.00 +SO-102026,2025-09-16,West,online,VE-1000,6,1971.00 +SO-102027,2025-09-16,South,partner,VE-2000,6,2430.00 +SO-102028,2025-09-16,West,online,VE-2000,8,3600.00 +SO-102029,2025-09-16,East,direct,VE-1000,15,5475.00 +SO-102030,2025-09-16,South,online,VE-2000,10,4500.00 +SO-102031,2025-09-16,South,partner,VE-2000,15,6750.00 +SO-102032,2025-09-16,North,direct,VE-2000,10,4500.00 +SO-102033,2025-09-16,South,online,VE-2000,8,3420.00 +SO-102034,2025-09-16,South,online,VE-1000,1,365.00 +SO-102035,09/16/2025,North,direct,VE-1000,11,4015.00 +SO-102036,2025-09-16,North,online,VE-2000,10,4500.00 +SO-102037,2025-09-16,East,direct,VE-1000,10,3650.00 +SO-102038,2025-09-16,East,partner,VE-3000,14,26467.00 +SO-102039,2025-09-16,East,direct,VE-1000,1,328.50 +SO-102040,2025-09-16,West,direct,VE-2000,15,6750.00 +SO-102041,2025-09-16,East,direct,VE-1000,15,5475.00 +SO-102042,2025-09-16,West,direct,VE-3000,1,1791.00 +SO-102043,2025-09-16,East,online,VE-2000,9,3847.50 +SO-102044,2025-09-16,East,online,VE-1000,7,2555.00 +SO-102045,2025-09-16,East,partner,VE-1000,1,365.00 +SO-102046,2025-09-16,South,online,VE-1000,13,4507.75 +SO-102047,2025-09-16,South,online,VE-1000,21,6898.50 +SO-102048,2025-09-16,North,online,VE-1000,9,2956.50 +SO-102049,2025-09-17,North,direct,VE-1000,5,1825.00 +SO-102050,2025-09-17,West,partner,VE-1000,18,6570.00 +SO-102051,2025-09-17,East,online,VE-3000,1,1791.00 +SO-102052,2025-09-17,East,partner,VE-1000,4,1314.00 +SO-102053,2025-09-17,South,online,VE-1000,16,5840.00 +SO-102054,2025-09-17,North,online,VE-1000,11,4015.00 +SO-102055,2025-09-17,South,partner,VE-2000,17,7650.00 +SO-102056,2025-09-17,South,direct,VE-2000,5,2025.00 +SO-102057,2025-09-17,East,partner,VE-2000,13,5265.00 +SO-102058,2025-09-17,West,direct,VE-1000,11,3814.25 +SO-102059,2025-09-17,West,partner,VE-3000,11,20795.50 +SO-102060,2025-09-17,West,partner,VE-1000,15,4927.50 +SO-102061,2025-09-17,North,partner,VE-1000,1,365.00 +SO-102062,2025-09-17,North,online,VE-2000,1,405.00 +SO-102063,2025-09-17,East,direct,VE-1000,8,2628.00 +SO-102064,2025-09-17,East,online,VE-2000,8,3240.00 +SO-102065,2025-09-17,South,partner,VE-1000,8,2774.00 +SO-102066,2025-09-17,East,online,VE-1000,14,5110.00 +SO-102067,2025-09-17,West,online,VE-2000,5,2250.00 +SO-102068,2025-09-17,West,online,VE-1000,10,3650.00 +SO-102069,09/17/2025,West,online,VE-2000,1,450.00 +SO-102070,2025-09-17,South,direct,VE-3000,6,11940.00 +SO-102071,2025-09-18,North,online,VE-2000,9,4050.00 +SO-102072,2025-09-18,North,online,VE-2000,10,4500.00 +SO-102073,2025-09-18,East,online,VE-1000,7,2555.00 +SO-102074,2025-09-18,North,online,VE-2000,3,1350.00 +SO-102075,2025-09-18,South,online,VE-1000,13,4745.00 +SO-102076,2025-09-18,North,direct,VE-3000,11,20795.50 +SO-102077,2025-09-18,East,direct,VE-3000,2,3980.00 +SO-102078,2025-09-18,West,direct,VE-1000,5,1825.00 +SO-102079,2025-09-18,East,online,VE-2000,13,5850.00 +SO-102080,2025-09-18,West,partner,VE-1000,5,1825.00 +SO-102081,2025-09-18,North,partner,VE-1000,6,2190.00 +SO-102082,2025-09-18,North,online,VE-2000,17,7650.00 +SO-102083,2025-09-18,North,partner,VE-3000,4,7960.00 +SO-102084,2025-09-18,North,direct,VE-1000,14,5110.00 +SO-102085,2025-09-18,East,online,VE-1000,6,1971.00 +SO-102086,2025-09-19,West,partner,VE-2000,1,450.00 +SO-102087,2025-09-19,South,direct,VE-2000,4,1800.00 +SO-102088,2025-09-19,South,partner,VE-2000,4,1710.00 +SO-102089,2025-09-19,West,online,VE-2000,8,3240.00 +SO-102090,2025-09-19,West,partner,VE-2000,4,1800.00 +SO-102091,2025-09-19,East,direct,VE-1000,2,693.50 +SO-102092,2025-09-19,North,partner,VE-1000,5,1825.00 +SO-102093,2025-09-19,East,direct,VE-2000,11,4950.00 +SO-102094,2025-09-19,South,online,VE-1000,16,5548.00 +SO-102095,2025-09-19,East,online,VE-3000,18,34029.00 +SO-102096,2025-09-19,North,online,VE-2000,13,5265.00 +SO-102097,2025-09-19,North,direct,VE-1000,10,3650.00 +SO-102098,2025-09-19,West,online,VE-1000,12,3942.00 +SO-102099,2025-09-19,North,online,VE-2000,8,3240.00 +SO-102100,2025-09-19,East,partner,VE-1000,9,3285.00 +SO-102101,2025-09-19,North,direct,VE-2000,15,6750.00 +SO-102102,2025-09-19,East,online,VE-3000,2,3980.00 +SO-102103,2025-09-19,South,partner,VE-1000,8,2920.00 +SO-102104,2025-09-19,East,partner,VE-2000,8,3240.00 +SO-102105,2025-09-20,North,online,VE-1000,1,328.50 +SO-102106,2025-09-20,East,direct,VE-1000,2,657.00 +SO-102107,2025-09-20,West,direct,VE-3000,7,13930.00 +SO-102108,2025-09-20,East,direct,VE-3000,11,21890.00 +SO-102109,2025-09-20,East,partner,VE-1000,19,6935.00 +SO-102110,2025-09-20,South,partner,VE-1000,12,4380.00 +SO-102111,2025-09-20,South,partner,VE-3000,11,19701.00 +SO-102112,2025-09-20,North,direct,VE-2000,7,2992.50 +SO-102113,2025-09-20,South,online,VE-1000,17,6205.00 +SO-102114,2025-09-20,North,partner,VE-2000,7,2992.50 +SO-102115,2025-09-20,South,partner,VE-3000,17,33830.00 +SO-102116,2025-09-20,West,direct,VE-1000,2,730.00 +SO-102117,2025-09-20,East,online,VE-1000,2,657.00 +SO-102118,2025-09-20,West,partner,VE-2000,5,2250.00 +SO-102119,2025-09-20,West,partner,VE-1000,7,2555.00 +SO-102120,2025-09-20,North,direct,VE-3000,11,19701.00 +SO-102121,2025-09-20,East,partner,VE-2000,1,450.00 +SO-102122,2025-09-20,North,online,VE-1000,8,2628.00 +SO-102123,2025-09-20,North,partner,VE-1000,22,8030.00 +SO-102124,2025-09-20,West,direct,VE-2000,1,450.00 +SO-102125,2025-09-20,North,online,VE-2000,10,4500.00 +SO-102126,2025-09-20,North,direct,VE-1000,10,3650.00 +SO-102127,2025-09-20,West,online,VE-1000,12,3942.00 +SO-102128,2025-09-20,West,partner,VE-2000,1,450.00 +SO-102129,2025-09-20,West,direct,VE-1000,7,2555.00 +SO-102130,2025-09-20,South,online,VE-1000,8,2920.00 +SO-102131,2025-09-20,South,partner,VE-3000,12,21492.00 +SO-102132,2025-09-20,West,direct,VE-1000,7,2555.00 +SO-102133,2025-09-20,South,direct,VE-2000,7,3150.00 +SO-102134,09/20/2025,South,partner,VE-2000,7,3150.00 +SO-102135,2025-09-20,South,direct,VE-2000,9,4050.00 +SO-102136,2025-09-20,West,partner,VE-1000,10,3285.00 +SO-102137,2025-09-20,East,partner,VE-2000,17,6885.00 +SO-102138,2025-09-21,North,direct,VE-3000,7,13930.00 +SO-102139,2025-09-21,North,direct,VE-1000,8,2920.00 +SO-102140,2025-09-21,West,direct,VE-2000,4,1800.00 +SO-102141,2025-09-21,West,direct,VE-1000,1,365.00 +SO-102142,2025-09-21,North,direct,VE-1000,5,1642.50 +SO-102143,2025-09-21,East,partner,VE-3000,1,1890.50 +SO-102144,09/21/2025,East,online,VE-3000,11,21890.00 +SO-102145,2025-09-21,South,partner,VE-1000,10,3467.50 +SO-102146,2025-09-21,North,online,VE-2000,1,427.50 +SO-102147,2025-09-21,South,partner,VE-1000,9,3120.75 +SO-102148,2025-09-21,South,partner,VE-2000,10,4500.00 +SO-102149,2025-09-21,North,online,VE-1000,11,4015.00 +SO-102150,2025-09-21,West,online,VE-1000,11,3613.50 +SO-102151,2025-09-21,East,partner,VE-1000,7,2555.00 +SO-102152,2025-09-21,North,partner,VE-1000,10,3650.00 +SO-102153,2025-09-21,North,direct,VE-1000,8,2774.00 +SO-102154,2025-09-21,North,direct,VE-1000,5,1825.00 +SO-102155,2025-09-21,South,partner,VE-2000,8,3600.00 +SO-102156,2025-09-21,East,direct,VE-1000,5,1642.50 +SO-102157,2025-09-21,South,direct,VE-1000,11,3814.25 +SO-102158,2025-09-21,East,partner,VE-3000,10,19900.00 +SO-102159,2025-09-21,North,direct,VE-2000,11,4950.00 +SO-102160,2025-09-21,West,partner,VE-1000,13,4745.00 +SO-102161,2025-09-21,East,online,VE-1000,1,365.00 +SO-102162,2025-09-21,South,direct,VE-2000,3, +SO-102163,2025-09-21,West,online,VE-1000,3,1095.00 +SO-102164,2025-09-21,West,direct,VE-1000,1,328.50 +SO-102165,2025-09-21,South,partner,VE-2000,15,6750.00 +SO-102166,2025-09-21,North,partner,VE-1000,3,1095.00 +SO-102167,2025-09-22,West,direct,VE-3000,21,37611.00 +SO-102168,2025-09-22,East,online,VE-2000,6,2700.00 +SO-102169,2025-09-22,North,partner,VE-3000,3,5671.50 +SO-102170,2025-09-22,West,partner,VE-3000,17,33830.00 +SO-102171,2025-09-22,West,online,VE-2000,8,3600.00 +SO-102172,2025-09-22,South,online,VE-1000,2,730.00 +SO-102173,2025-09-22,East,direct,VE-1000,13,4745.00 +SO-102174,2025-09-22,South,direct,VE-1000,8,2920.00 +SO-102175,2025-09-22,North,partner,VE-1000,5,1825.00 +SO-102176,2025-09-22,East,partner,VE-2000,7,3150.00 +SO-102177,2025-09-22,South,direct,VE-1000,7,2299.50 +SO-102178,2025-09-22,East,online,VE-2000,9,3847.50 +SO-102179,2025-09-22,South,direct,VE-1000,16,5256.00 +SO-102180,2025-09-22,North,online,VE-2000,12,5400.00 +SO-102181,2025-09-22,West,partner,VE-2000,11,4950.00 +SO-102182,2025-09-22,North,direct,VE-3000,9,16119.00 +SO-102183,2025-09-22,West,direct,VE-1000,19,6241.50 +SO-102184,2025-09-22,South,direct,VE-2000,1,450.00 +SO-102185,2025-09-22,East,partner,VE-1000,12,4161.00 +SO-102186,2025-09-22,South,partner,VE-1000,7,2555.00 +SO-102187,2025-09-22,South,online,VE-2000,15,6412.50 +SO-102188,2025-09-22,North,online,VE-2000,10,4500.00 +SO-102189,2025-09-22,South,partner,VE-2000,1,427.50 +SO-102190,2025-09-22,East,online,VE-1000,11,4015.00 +SO-102191,2025-09-22,North,direct,VE-3000,13,23283.00 +SO-102192,2025-09-22,North,partner,VE-3000,7,13930.00 +SO-102193,2025-09-22,North,direct,VE-1000,2,730.00 +SO-102194,2025-09-22,East,online,VE-1000,1,365.00 +SO-102195,2025-09-22,East,direct,VE-3000,3,5671.50 +SO-102196,2025-09-22,North,online,VE-2000,13,5850.00 +SO-102197,2025-09-22,West,partner,VE-1000,1,365.00 +SO-102198,2025-09-23,South,online,VE-1000,10,3285.00 +SO-102199,2025-09-23,South,partner,VE-1000,2,730.00 +SO-102200,2025-09-23,South,partner,VE-1000,8,2920.00 +SO-102201,2025-09-23,North,partner,VE-1000,21,7665.00 +SO-102202,2025-09-23,North,partner,VE-1000,7,2427.25 +SO-102203,2025-09-23,East,partner,VE-2000,1,427.50 +SO-102204,2025-09-23,South,partner,VE-1000,16,5840.00 +SO-102205,2025-09-23,North,online,VE-1000,9,3120.75 +SO-102206,2025-09-23,North,partner,VE-1000,5,1642.50 +SO-102207,2025-09-23,West,direct,VE-1000,1,365.00 +SO-102208,09/23/2025,East,online,VE-1000,5,1825.00 +SO-102209,2025-09-23,West,online,VE-1000,13,4270.50 +SO-102210,2025-09-23,South,direct,VE-1000,9,3285.00 +SO-102211,2025-09-23,East,online,VE-1000,5,1642.50 +SO-102212,2025-09-23,North,direct,VE-2000,1,450.00 +SO-102213,2025-09-23,South,direct,VE-1000,3,1095.00 +SO-102214,2025-09-23,South,online,VE-1000,15,5201.25 +SO-102215,2025-09-23,North,direct,VE-1000,8,2774.00 +SO-102216,2025-09-23,North,partner,VE-2000,1,450.00 +SO-102217,2025-09-23,North,direct,VE-1000,6,2190.00 +SO-102218,2025-09-23,South,partner,VE-2000,1,427.50 +SO-102219,2025-09-24,South,direct,VE-1000,3,1095.00 +SO-102220,2025-09-24,West,partner,VE-3000,22,43780.00 +SO-102221,2025-09-24,East,direct,VE-1000,4,1460.00 +SO-102222,2025-09-24,South,direct,VE-1000,5,1825.00 +SO-102223,2025-09-24,West,direct,VE-3000,5,8955.00 +SO-102224,2025-09-24,West,direct,VE-2000,7,3150.00 +SO-102225,2025-09-24,North,online,VE-3000,6,11940.00 +SO-102226,2025-09-24,North,partner,VE-1000,6,1971.00 +SO-102227,2025-09-24,West,direct,VE-2000,9,4050.00 +SO-102228,2025-09-24,North,direct,VE-1000,8,2920.00 +SO-102229,2025-09-24,East,online,VE-1000,19,6588.25 +SO-102230,2025-09-24,North,online,VE-3000,1,1990.00 +SO-102231,2025-09-24,South,partner,VE-1000,8,2920.00 +SO-102232,2025-09-24,East,partner,VE-1000,14,5110.00 +SO-102233,2025-09-24,East,partner,VE-1000,15,5475.00 +SO-102234,2025-09-24,East,partner,VE-3000,3,5970.00 +SO-102235,2025-09-24,South,direct,VE-3000,9,17910.00 +SO-102236,2025-09-24,South,online,VE-1000,20,7300.00 +SO-102237,2025-09-24,East,online,VE-3000,9,17014.50 +SO-102238,2025-09-24,East,partner,VE-1000,13,4270.50 +SO-102239,2025-09-24,East,partner,VE-2000,8,3600.00 +SO-102240,2025-09-24,South,partner,VE-2000,25,10125.00 +SO-102241,2025-09-24,North,direct,VE-1000,10,3467.50 +SO-102242,2025-09-24,North,online,VE-2000,6,2565.00 +SO-102243,2025-09-24,South,online,VE-2000,2,900.00 +SO-102244,2025-09-24,West,partner,VE-1000,13,4745.00 +SO-102245,2025-09-25,North,online,VE-3000,6,11940.00 +SO-102246,2025-09-25,South,online,VE-1000,12,4161.00 +SO-102247,2025-09-25,East,partner,VE-2000,1,450.00 +SO-102248,2025-09-25,West,direct,VE-1000,10,3650.00 +SO-102249,2025-09-25,South,online,VE-1000,1,328.50 +SO-102250,2025-09-25,East,direct,VE-2000,11,4702.50 +SO-102251,2025-09-25,North,partner,VE-2000,4,1800.00 +SO-102252,2025-09-25,North,partner,VE-2000,18,8100.00 +SO-102253,2025-09-25,South,direct,VE-1000,4,1460.00 +SO-102254,2025-09-25,South,online,VE-2000,18,7695.00 +SO-102255,2025-09-25,East,online,VE-3000,11,21890.00 +SO-102256,2025-09-25,North,direct,VE-1000,8,2774.00 +SO-102257,2025-09-25,North,partner,VE-1000,3,1095.00 +SO-102258,2025-09-25,North,direct,VE-2000,3,1215.00 +SO-102259,2025-09-25,West,online,VE-3000,17,33830.00 +SO-102260,2025-09-25,South,direct,VE-1000,15,5475.00 +SO-102261,2025-09-25,North,online,VE-2000,13,5557.50 +SO-102262,2025-09-25,North,direct,VE-1000,10,3650.00 +SO-102263,2025-09-25,East,online,VE-1000,19,6935.00 +SO-102264,2025-09-25,South,online,VE-2000,10,4500.00 +SO-102265,2025-09-25,North,online,VE-1000,10,3650.00 +SO-102266,2025-09-25,North,partner,VE-3000,7,13930.00 +SO-102267,2025-09-25,East,direct,VE-1000,7,2555.00 +SO-102268,2025-09-25,North,direct,VE-1000,5,1733.75 +SO-102269,2025-09-25,North,direct,VE-2000,1,450.00 +SO-102270,2025-09-25,North,partner,VE-1000,11,3814.25 +SO-102271,2025-09-26,South,direct,VE-1000,3,1095.00 +SO-102272,2025-09-26,South,online,VE-1000,9,2956.50 +SO-102273,2025-09-26,West,direct,VE-3000,14,25074.00 +SO-102274,2025-09-26,West,direct,VE-3000,5,9950.00 +SO-102275,2025-09-26,North,online,VE-2000,3,1282.50 +SO-102276,2025-09-26,South,partner,VE-1000,12,4380.00 +SO-102277,2025-09-26,East,direct,VE-1000,3,1040.25 +SO-102278,2025-09-26,East,partner,VE-1000,4,1460.00 +SO-102279,2025-09-26,East,partner,VE-1000,12,3942.00 +SO-102280,2025-09-26,East,direct,VE-1000,10,3650.00 +SO-102281,2025-09-26,North,partner,VE-2000,8,3420.00 +SO-102282,2025-09-26,North,direct,VE-2000,1,427.50 +SO-102283,2025-09-26,West,partner,VE-2000,2,900.00 +SO-102284,2025-09-26,East,partner,VE-2000,4,1710.00 +SO-102285,09/26/2025,North,direct,VE-1000,10,3285.00 +SO-102286,2025-09-26,West,partner,VE-2000,17,7650.00 +SO-102287,2025-09-26,South,online,VE-2000,10,4500.00 +SO-102288,2025-09-26,North,direct,VE-1000,14,5110.00 +SO-102289,2025-09-26,North,online,VE-1000,1,346.75 +SO-102290,2025-09-26,North,online,VE-1000,1,328.50 +SO-102291,2025-09-26,South,online,VE-1000,1,365.00 +SO-102292,2025-09-26,West,partner,VE-2000,9,3645.00 +SO-102293,2025-09-26,South,online,VE-3000,7,13930.00 +SO-102294,2025-09-26,North,direct,VE-1000,7,2555.00 +SO-102295,2025-09-26,East,direct,VE-2000,11,4950.00 +SO-102296,2025-09-26,West,direct,VE-1000,16,5840.00 +SO-102297,2025-09-26,East,partner,VE-1000,8,2920.00 +SO-102298,2025-09-26,North,direct,VE-2000,14,6300.00 +SO-102299,2025-09-26,South,partner,VE-2000,10,4500.00 +SO-102300,2025-09-26,South,online,VE-1000,10,3650.00 +SO-102301,2025-09-27,West,direct,VE-3000,10,17910.00 +SO-102302,2025-09-27,West,direct,VE-1000,9,3120.75 +SO-102303,2025-09-27,East,direct,VE-2000,1,450.00 +SO-102304,2025-09-27,East,partner,VE-1000,11,3613.50 +SO-102305,2025-09-27,North,partner,VE-1000,6,1971.00 +SO-102306,2025-09-27,North,direct,VE-1000,6,2190.00 +SO-102307,2025-09-27,North,partner,VE-2000,1,450.00 +SO-102308,2025-09-27,South,online,VE-1000,4,1460.00 +SO-102309,2025-09-27,South,direct,VE-1000,6,2190.00 +SO-102310,2025-09-27,North,partner,VE-1000,16,5840.00 +SO-102311,2025-09-27,South,partner,VE-2000,11,4950.00 +SO-102312,2025-09-27,East,partner,VE-1000,8,2628.00 +SO-102313,2025-09-27,North,partner,VE-1000,10,3650.00 +SO-102314,2025-09-27,East,direct,VE-3000,1,1990.00 +SO-102315,2025-09-27,North,partner,VE-3000,4,7960.00 +SO-102316,2025-09-27,South,direct,VE-1000,5,1825.00 +SO-102317,2025-09-27,North,partner,VE-1000,7,2427.25 +SO-102318,2025-09-27,South,partner,VE-1000,7,2555.00 +SO-102319,2025-09-27,West,partner,VE-3000,1,1890.50 +SO-102320,2025-09-27,North,partner,VE-1000,11,4015.00 +SO-102321,2025-09-27,South,direct,VE-2000,12,4860.00 +SO-102322,2025-09-27,SOUTH,partner,VE-3000,6,11343.00 +SO-102323,2025-09-27,West,direct,VE-3000,3,5671.50 +SO-102324,2025-09-27,South,partner,VE-1000,19,6588.25 +SO-102325,2025-09-27,North,partner,VE-3000,6,11343.00 +SO-102326,2025-09-27,North,direct,VE-2000,16,6480.00 +SO-102327,09/27/2025,West,online,VE-2000,4,1800.00 +SO-102328,2025-09-27,East,online,VE-1000,1,365.00 +SO-102329,2025-09-27,North,direct,VE-1000,10,3650.00 +SO-102330,2025-09-27,East,online,VE-1000,8,2628.00 +SO-102331,2025-09-28,East,direct,VE-2000,1,427.50 +SO-102332,2025-09-28,South,online,VE-1000,3,1040.25 +SO-102333,2025-09-28,West,online,VE-1000,1,365.00 +SO-102334,2025-09-28,North,direct,VE-3000,6,11940.00 +SO-102335,2025-09-28,East,online,VE-1000,5,1825.00 +SO-102336,2025-09-28,East,partner,VE-1000,10,3650.00 +SO-102337,2025-09-28,East,partner,VE-1000,16,5548.00 +SO-102338,2025-09-28,West,direct,VE-1000,14,4854.50 +SO-102339,09/28/2025,West,online,VE-2000,7,2835.00 +SO-102340,2025-09-28,North,partner,VE-1000,5,1825.00 +SO-102341,2025-09-28,North,online,VE-1000,1,346.75 +SO-102342,2025-09-28,West,direct,VE-3000,10,19900.00 +SO-102343,2025-09-28,East,partner,VE-2000,1,450.00 +SO-102344,2025-09-28,North,partner,VE-3000,7,12537.00 +SO-102345,2025-09-28,South,partner,VE-1000,6,2190.00 +SO-102346,2025-09-28,North,partner,VE-1000,14,5110.00 +SO-102347,2025-09-28,East,online,VE-2000,18,7695.00 +SO-102348,2025-09-28,South,partner,VE-1000,17,6205.00 +SO-102349,2025-09-28,South,online,VE-1000,13,4745.00 +SO-102350,2025-09-28,East,direct,VE-3000,1,1990.00 +SO-102351,2025-09-28,North,partner,VE-2000,6,2700.00 +SO-102352,2025-09-28,South,direct,VE-1000,8,2920.00 +SO-102353,2025-09-28,South,partner,VE-2000,16,6480.00 +SO-102354,2025-09-28,North,partner,VE-2000,17,7650.00 +SO-102355,2025-09-28,South,online,VE-1000,10,3467.50 +SO-102356,2025-09-28,North,partner,VE-3000,4,7960.00 +SO-102357,2025-09-29,North,partner,VE-1000,4,1460.00 +SO-102358,2025-09-29,West,online,VE-2000,1,450.00 +SO-102359,2025-09-29,West,partner,VE-1000,1,365.00 +SO-102360,2025-09-29,North,online,VE-1000,6,2190.00 +SO-102361,2025-09-29,East,direct,VE-1000,13,4507.75 +SO-102362,2025-09-29,East,online,VE-1000,8,2920.00 +SO-102363,2025-09-29,South,direct,VE-1000,10,3650.00 +SO-102364,2025-09-29,West,partner,VE-1000,5,1825.00 +SO-102365,2025-09-29,East,direct,VE-3000,6,11940.00 +SO-102366,2025-09-29,South,partner,VE-1000,9,2956.50 +SO-102367,2025-09-29,North,partner,VE-1000,8,2920.00 +SO-102368,2025-09-29,West,online,VE-2000,12,4860.00 +SO-102369,2025-09-29,East,direct,VE-3000,2,3781.00 +SO-102370,2025-09-29,East,online,VE-2000,21,8977.50 +SO-102371,2025-09-29,North,partner,VE-1000,4,1314.00 +SO-102372,2025-09-29,South,direct,VE-2000,7,2835.00 +SO-102373,2025-09-29,East,online,VE-2000,1,450.00 +SO-102374,2025-09-29,West,partner,VE-3000,14,26467.00 +SO-102375,2025-09-29,East,direct,VE-1000,4,1460.00 +SO-102376,2025-09-29,North,partner,VE-2000,18,8100.00 +SO-102377,2025-09-29,North,direct,VE-2000,7,3150.00 +SO-102378,2025-09-29,North,online,VE-1000,4,1460.00 +SO-102379,2025-09-29,South,partner,VE-1000,9,3120.75 +SO-102380,2025-09-29,East,online,VE-3000,14,27860.00 +SO-102381,2025-09-29,East,direct,VE-1000,7,2555.00 +SO-102382,2025-09-29,East,direct,VE-1000,12,3942.00 +SO-102383,2025-09-29,North,partner,VE-1000,8,2920.00 +SO-102384,2025-09-30,East,partner,VE-2000,12,5130.00 +SO-102385,2025-09-30,South,online,VE-1000,7,2555.00 +SO-102386,2025-09-30,East,partner,VE-2000,1,450.00 +SO-102387,2025-09-30,East,partner,VE-3000,12,22686.00 +SO-102388,2025-09-30,East,online,VE-1000,10,3285.00 +SO-102389,2025-09-30,East,partner,VE-2000,9,3847.50 +SO-102390,2025-09-30,North,partner,VE-1000,10,3650.00 +SO-102391,2025-09-30,North,direct,VE-2000,13,5850.00 +SO-102392,2025-09-30,South,direct,VE-1000,20,7300.00 +SO-102393,2025-09-30,South,partner,VE-2000,12,5400.00 +SO-102394,2025-09-30,South,partner,VE-3000,10,19900.00 +SO-102395,2025-09-30,South,online,VE-3000,6,11940.00 +SO-102396,2025-09-30,East,partner,VE-1000,12,4380.00 +SO-102397,2025-09-30,West,online,VE-1000,18,5913.00 +SO-102398,2025-09-30,North,direct,VE-1000,4,1314.00 +SO-102399,2025-09-30,South,partner,VE-3000,15,29850.00 +SO-102400,2025-09-30,West,direct,VE-2000,4,1620.00 diff --git a/examples/codex_data_analysis/main.py b/examples/codex_data_analysis/main.py new file mode 100644 index 000000000..a29d1a67c --- /dev/null +++ b/examples/codex_data_analysis/main.py @@ -0,0 +1,256 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A `runtime="codex"` analyst that writes, runs and *debugs* its own script. + +This is the task the codex runtime exists for. The agent is handed a 2 400-row +warehouse extract with realistic defects buried in it — blank revenue cells, +prices carrying thousands separators, a second date format, three spellings of +one region — none of them in the first 200 rows and none of them mentioned in +the prompt. The file is far too big to eyeball, so the agent has to write an +analysis script, run it, read the traceback, fix it, and run it again until the +numbers come out. On ``runtime="adk"`` the model can only *emit* a script and +hope; here it executes one inside an OS sandbox and sees what happened. + +What the example demonstrates: + +- **Path-passing, not payload-passing.** ``fetch_sales_extract`` lands the CSV + in Codex's workspace and returns ``{"path": ..., "rows": 2400}``. ADK tool + results are executed by the runtime's shim and come back to the model as text + in its context, so the workspace is the data plane and tool results are the + control plane. See ``analytics_tools.py``. +- **Iteration on real errors.** Watch the ``exec_command`` lines below: the + first ``python3`` run fails, and the agent recovers from the traceback. +- **The sandbox as the security boundary.** ``sandbox="workspace_write"`` + + ``network_access=False`` + ``approval_mode="deny_all"`` means the model may + compute anything it likes over the data but has no way to send it anywhere. + The audited ``publish_report`` tool is the only outbound path, and ``outbox/`` + sits outside the workspace precisely so the sandbox cannot write to it. +- **A workspace that survives the turn.** Turn 2 asks for a different chart; + the extract, the script and the report are all still there, so the agent + edits instead of starting over. +- **A skill for the house format.** ``skills/sales-report/SKILL.md`` is driven + by Codex's native skill system, so the report layout stays out of the prompt. + +Run: + python examples/codex_data_analysis/main.py + +Requires: +- ``pip install "veadk-python[codex]"`` (openai-codex plus the bundled Codex + CLI binary). macOS or Linux — the OS sandbox is seatbelt / landlock+seccomp. +- Ark (or another OpenAI-compatible chat) credentials via ``MODEL_AGENT_API_KEY`` + / ``MODEL_AGENT_API_BASE`` / ``MODEL_AGENT_NAME`` (see ``.env.example``). +""" + +import asyncio +import os +import shutil +from pathlib import Path +from typing import Any + +from google.adk.agents import RunConfig +from google.adk.skills import load_skill_from_dir +from google.adk.tools.skill_toolset import SkillToolset +from google.genai import types + +from analytics_tools import ( + OUTBOX, + fetch_sales_extract, + last_seen_workspace, + publish_report, +) +from veadk import Agent, Runner +from veadk.memory.short_term_memory import ShortTermMemory +from veadk.runtime.codex import CodexRuntimeConfig + +_HERE = Path(__file__).resolve().parent +_SKILL_DIR = _HERE / "skills" / "sales-report" + +_SESSION_ID = "q3-review" + +INSTRUCTION = """\ +You are a revenue analyst. You work inside a sandboxed workspace: you may +create and run files there, but you have no network access and only the Python +standard library — pandas, numpy and matplotlib are not installed and cannot be +installed. + +Act, do not narrate. Never end a message with a plan you have not carried out: +a reply that contains no tool call ends your turn and the work stops there. + +How you work: + +1. Call `fetch_sales_extract` to land the quarter's extract in your workspace. + It returns a path and a row count, never the rows themselves — read the file + with your own code. +2. Write an analysis script and *run* it with `python3`. Never report a number + you have not computed by executing code. If a run fails, read the error, fix + the script, and run it again — repeat until it succeeds. +3. Write the report to `report.md` and the chart to `chart.svg`, following the + sales-report skill. +4. Call `publish_report` with both paths — it is the only way a file leaves the + sandbox. Your turn is finished only once it returns status "ok". + +Reply only then, with a short plain-language summary of what the numbers say, +plus anything a human reading the report should know about how you got there. +""" + +TURNS = ( + "Produce the 2025Q3 sales review and publish it.", + "Reviewers want the trend chart replaced: make chart.svg a horizontal bar " + "chart of revenue by region, highest first. Leave the rest of the report " + "as it is and publish the updated version.", +) + + +def build_agent() -> Agent: + """Build the analyst agent, sandbox settings included.""" + # The house report format, loaded the ADK-native way. The codex runtime + # materializes it into Codex's own skill directory. + report_skill = SkillToolset(skills=[load_skill_from_dir(str(_SKILL_DIR))]) + + return Agent( + name="codex_data_analyst", + description="Turns a raw sales extract into a published report.", + instruction=INSTRUCTION, + runtime="codex", + model_name=os.getenv("MODEL_AGENT_NAME", "deepseek-v4-flash-260425"), + model_api_base=os.getenv( + "MODEL_AGENT_API_BASE", "https://ark.cn-beijing.volces.com/api/v3" + ), + model_api_key=os.getenv("MODEL_AGENT_API_KEY", ""), + tools=[report_skill, fetch_sales_extract, publish_report], + codex_runtime_config=CodexRuntimeConfig( + # The model may write and run code, but only inside the workspace. + sandbox="workspace_write", + # Honoured by workspace_write: no sockets from inside the sandbox. + # With this off, the two ADK tools are the only way in or out. + network_access=False, + # Refuse every escalation Codex asks for. Never use "auto_review" + # here: it is full auto-approval, not a review step. + approval_mode="deny_all", + # `workspace_root` and `reuse_workspace` are deliberately unset: + # each (app, user, session, agent) then gets its own directory, + # which still survives across the turns of that session. The two + # ADK tools find it with `current_workspace()` — see + # `analytics_tools.py`. Pinning would collapse every session onto + # one directory; the README says when that is worth doing. + # + # ADK tool round-trips allowed for the whole turn. This agent needs + # two (fetch + publish); the rest of the budget is for retries. + max_tool_iterations=8, + tool_timeout_seconds=120.0, + ), + ) + + +def _reset_outbox() -> None: + """Empty the outbox so the run's published files are the only ones in it. + + The workspace needs no such reset: it is created per session under the + runtime's own temporary root, so this process starts with an empty one. + """ + shutil.rmtree(OUTBOX, ignore_errors=True) + OUTBOX.mkdir(parents=True, exist_ok=True) + + +def _truncate(value: Any, limit: int = 160) -> str: + text = " ".join(str(value).split()) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _print_event(event: Any) -> None: + """Render one ADK event: sandboxed commands, ADK tool calls, final text. + + ``exec_command`` is Codex running something inside the sandbox — the + runtime surfaces it as an ordinary ADK function-call event, which is what + makes the write / run / fix / re-run loop visible from here. + """ + if event.partial: + return + + for call in event.get_function_calls() or []: + args = call.args or {} + if call.name == "exec_command": + print(f" $ {_truncate(args.get('command', ''))}") + else: + print(f" → {call.name}({_truncate(args, 100)})") + + for response in event.get_function_responses() or []: + payload = response.response + if response.name == "exec_command": + exit_code = (payload or {}).get("exit_code") + if exit_code: # non-zero, and not None (still running) + tail = [ + line + for line in str((payload or {}).get("output", "")).splitlines() + if line.strip() + ][-2:] + print(f" exit={exit_code}") + for line in tail: + print(f" | {_truncate(line)}") + else: + print(f" ← {response.name}: {_truncate(payload, 200)}") + + for part in event.content.parts if event.content and event.content.parts else []: + if part.text and not part.thought: + print(f"\nAgent: {part.text.strip()}\n") + + +def _print_tree(label: str, root: Path) -> None: + print(f"\n{label} ({root}):") + entries = sorted(p for p in root.rglob("*") if p.is_file()) + if not entries: + print(" (empty)") + for path in entries: + print(f" {path.relative_to(root)} {path.stat().st_size} B") + + +async def main() -> None: + _reset_outbox() + + runner = Runner(agent=build_agent(), short_term_memory=ShortTermMemory()) + await runner.short_term_memory.create_session( + app_name=runner.app_name, user_id=runner.user_id, session_id=_SESSION_ID + ) + + # A hard cost ceiling for the turn. The shim charges this budget before + # every backend call, so unlike other external runtimes codex enforces it + # exactly — raise it if a turn ends with LlmCallsLimitExceededError. + run_config = RunConfig(max_llm_calls=60) + + for number, prompt in enumerate(TURNS, start=1): + print(f"\n{'=' * 72}\nTurn {number} — User: {prompt}\n{'=' * 72}") + async for event in runner.run_async( + user_id=runner.user_id, + session_id=_SESSION_ID, + new_message=types.Content(role="user", parts=[types.Part(text=prompt)]), + run_config=run_config, + ): + _print_event(event) + + # The workspace is the agent's scratch space: its script and its drafts are + # still there. Printed here rather than left for you to `ls`, because the + # runtime removes its per-session workspaces when the process exits — pin + # `workspace_root` + `reuse_workspace` if you want to poke at one later. + # The outbox holds only what publish_report let through, and it persists. + workspace = last_seen_workspace() + if workspace is None: + print("\nWorkspace: no tool ran, so nothing reported a workspace.") + else: + _print_tree("Workspace", workspace) + _print_tree("Outbox", OUTBOX) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/codex_data_analysis/skills/sales-report/SKILL.md b/examples/codex_data_analysis/skills/sales-report/SKILL.md new file mode 100644 index 000000000..463ed72cd --- /dev/null +++ b/examples/codex_data_analysis/skills/sales-report/SKILL.md @@ -0,0 +1,54 @@ +--- +name: sales-report +description: The house format for a quarterly sales review. Use whenever you are asked to produce a sales, revenue or bookings report. +--- + +A sales review is one Markdown file with exactly these sections, in this order. + +```markdown +# Sales Review + +## Headline + +- **Total revenue**: CNY +- **Total units**: +- **Top region**: (CNY ) + +## By region + +| Region | Revenue (CNY) | Units | Share | +| --- | ---: | ---: | ---: | +| ... | ... | ... | ... | + +Sorted by revenue, highest first. Share is a percentage of total revenue with +one decimal. The last row is a **Total** row. + +## Trend + +![Revenue by month](chart.svg) + +One sentence naming the strongest and weakest month. Unless the request says +otherwise, the chart is revenue by month. + +## Data notes + +- One bullet per data-quality problem you had to correct, naming the affected + column, how many rows it hit, and what you did about it. +- `- none` if there were none. +``` + +## Chart rules + +No plotting library is available, so write the SVG by hand: + +- at most 640×320, with `viewBox`, and no external fonts, images or CSS; +- one axis line, every bar or point labelled with its value; +- a single accent colour plus a grey axis — no gradients; +- round numbers on the axis, not raw maxima. + +## Rules + +- Every figure in the report comes from the extract. Never carry a number over + from a previous draft — recompute it. +- Money is CNY, two decimals, thousands-separated. Units are integers. +- Keep the whole report under 60 lines. diff --git a/examples/codex_ops_assistant/.env.example b/examples/codex_ops_assistant/.env.example new file mode 100644 index 000000000..71b0fab3e --- /dev/null +++ b/examples/codex_ops_assistant/.env.example @@ -0,0 +1,10 @@ +# Copy this file to `.env` and fill in your key. + +# --- Agent reasoning model (Volcengine Ark) --- +# Get an API key at https://console.volcengine.com/ark +# Pick a model with solid tool-calling and code-writing ability: this agent +# writes and debugs its own analysis scripts. +MODEL_AGENT_PROVIDER=openai +MODEL_AGENT_NAME=deepseek-v4-pro-260425 +MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 +MODEL_AGENT_API_KEY=your-ark-api-key-here diff --git a/examples/codex_ops_assistant/.gitignore b/examples/codex_ops_assistant/.gitignore new file mode 100644 index 000000000..b93f13d08 --- /dev/null +++ b/examples/codex_ops_assistant/.gitignore @@ -0,0 +1,5 @@ +# Generated at run time: the simulated backend's storage and the tickets the +# agent filed. Codex's workspace is no longer here -- the runtime keeps it +# under its own per-session temp root, which `current_workspace()` resolves. +_store/ +outbox/ diff --git a/examples/codex_ops_assistant/README.md b/examples/codex_ops_assistant/README.md new file mode 100644 index 000000000..f142dd9e0 --- /dev/null +++ b/examples/codex_ops_assistant/README.md @@ -0,0 +1,358 @@ +# Ops triage assistant (`runtime="codex"`) + +> 中文版见 [README.zh.md](./README.zh.md) + +**The model can read a day of production logs and physically cannot leak +them.** It runs in an OS sandbox with `network_access=False` — there is no +socket to send anything out of — and it can write only its own scratch +directory. The single channel to the outside world is `file_incident_ticket`, +an ADK tool you wrote, whose arguments you can log, validate and cap. + +That is the answer to "can I let an LLM near my logs", and it is why this +example exists. + +What it does: an on-call agent investigates an incident it has never seen +before. ADK tools pull raw logs, a metric series and the deploy log out of an +internal system and drop them in the sandbox as files. Codex then writes and +runs its own shell and Python to grep, aggregate and correlate them — +several programs, refining as it learns the shape of the data — and finally +files a ticket naming a root cause with the numbers behind it. + +``` + ┌── ADK tools (your code, your process) ──┐ + internal │ fetch_application_logs │ workspace/ + systems ───► │ fetch_service_metrics write files ──┼───► logs/*.log + │ fetch_deploy_history │ metrics/*.csv + └─────────────────────────────────────────┘ deploys/*.json + │ + ┌────────────────────────────────────────────┘ + ▼ + Codex, inside the OS sandbox: no network, no writes outside + the workspace. Writes analysis/*.py, runs them, reads the + output, writes the next one. + │ + ▼ file_incident_ticket(...) ← the only egress + outbox/INC-....json (outside the workspace, + unreachable from the sandbox) +``` + +## Why this task suits `codex` and not `adk` + +Be honest about this: for most agents, `runtime="codex"` is strictly worse than +the default `runtime="adk"`. It spawns a subprocess per turn, re-serializes the +conversation into the prompt, and refuses a long list of ADK features. It earns +its cost only when the agent's real work is *running code it just wrote*. Ad-hoc +log analysis is that case: + +- **You cannot pre-build a tool per question.** "Which error signature changed + rate after 14:00?" "Was the pool saturated before or after the latency rose?" + "Does this pattern appear a week earlier?" The useful aggregations over a log + file are unbounded, and each new incident asks a new one. A tool per question + is a treadmill; an interpreter is not. +- **The data does not fit in the context, and the model cannot count.** One day + is 7,494 log lines (1.4 MB) plus 8,640 metric rows. Even if you paid to paste + them in, "951 payment timeouts vs 640 pool timeouts" is a job for + `collections.Counter`, not for a language model reading. Executed code gives + exact numbers; a summarized prompt gives plausible ones. +- **Getting it right takes several passes.** In our run the agent looked at the + head of the log, discovered it is not uniformly JSON, wrote a per-hour + signature histogram, then a change-point script, then a metrics script, then + a saturation check — each informed by the last. That loop is Codex's native + mode: many commands per model turn, with the program output staying inside + its loop instead of round-tripping through your event stream. +- **Sandboxing is what makes it acceptable at all.** Letting a model write and + execute arbitrary code over production logs is only sane if the code cannot + reach the network or the rest of the disk. That is a runtime property, not + something an instruction can promise. + +The nearest alternative is `runtime="adk"` with an ADK `code_executor`. +Compare honestly: `UnsafeLocalCodeExecutor` runs model-written code in your own +process; `ContainerCodeExecutor` needs a Docker daemon you operate; +`VertexAiCodeExecutor` is a managed Google service. The codex runtime gives you +an OS-level sandbox (macOS seatbelt / Linux landlock+seccomp) on the machine +you already have, a workspace that survives the whole session, and an agentic +loop that runs many commands per model turn rather than one code block per LLM +call. + +### When *not* to reach for this runtime + +- **The question is fixed.** If on-call always asks "how many 5xx yesterday", + write that tool. It is cheaper, faster, deterministic, and testable. +- **You need ADK features this runtime replaces.** `veadk/runtime/compat.py` + refuses `sub_agents`, `model=` (use `model_name=`), `output_schema`, + `planner`, `code_executor`, `include_contents="none"` and + `enable_supervisor`, and warns that `knowledgebase`, `example_store`, + model fallback chains, per-LLM-call callbacks and per-call tracing spans are + dropped. Read that file before porting an agent. +- **Per-turn latency or cost is the constraint.** Each turn spawns a Codex + subprocess, and the sandboxed loop makes many backend calls inside one turn. +- **The data must never touch this host.** The workspace is a real directory on + the machine running the agent. + +## The rule that makes ADK tools work here + +**The workspace is the data plane. Tool arguments and results are the control +plane.** + +An ADK tool under `runtime="codex"` does not run inside the sandbox. VeADK's +shim executes it in your process and pastes whatever it returns into the +model's context as the function-call result. A tool that returns 7,494 log +lines therefore does not "give the model the logs" — it destroys the turn, and +the model still has to grep the text afterwards. + +So every `fetch_*` tool here writes a file into the workspace and returns a +receipt: + +```python +{"status": "ok", + "path": "logs/checkout-api-prod_20260824T0000_20260825T0000.log", + "lines": 7494, "bytes": 1396879, + "note": "one event per line, chronological; content not returned"} +``` + +Roughly 40 tokens instead of 400,000. The model gets a pointer; the sandbox +gets the bytes. This is the single most important thing to get right when +combining ADK tools with this runtime, in either direction: + +- **Tools hand data *in* by writing files** and returning a path plus enough + shape (row count, column names, units) to plan against. +- **The model hands conclusions *out* through tool arguments**, which are + small, structured, and yours to validate — `file_incident_ticket` caps field + lengths, checks the severity enum, and prints one audit line per ticket. + +### Where the tools write + +An ADK tool asks for the directory of the turn that is calling it: + +```python +from veadk.runtime.codex import current_workspace + +def fetch_application_logs(stream: str, start_time: str, end_time: str) -> dict: + workspace = current_workspace() # this turn's directory, or None + if workspace is None: # not a codex turn — an error, not a guess + return {"status": "error", "message": "no sandbox working directory"} + ... +``` + +The runtime binds that value around each tool call, so it stays correct with +several sessions in flight in one process. That is why this example leaves +`workspace_root` and `reuse_workspace` unset: every `(app, user, session, +agent)` gets its own directory — which is what an on-call service wants, since +two incidents under investigation at the same time must not share a scratch +directory — and it still persists across the turns of one session, which is what +turn 2 reuses. + +`current_workspace()` returns `None` rather than raising outside a codex turn +(another runtime, an `AgentTool`, a unit test), and these tools turn that into +an ordinary error result the model can act on. + +Pinning is the opposite trade, and it is a **single-tenant** one: `workspace_root` +plus `reuse_workspace=True` gives you one predictable directory you can `ls` +after the process exits, at the price of every session sharing it. + +## The security configuration, knob by knob + +```python +codex_runtime_config=CodexRuntimeConfig( + sandbox="workspace_write", + network_access=False, + approval_mode="deny_all", + max_tool_iterations=12, + tool_timeout_seconds=60.0, +) +... +run_config=RunConfig(max_llm_calls=40) +``` + +| Setting | What it prevents | +| --- | --- | +| `sandbox="workspace_write"` | Model-written code can create and overwrite files **only** in the workspace (plus the temp dirs). It cannot touch your source tree, your dotfiles, or `outbox/`. Codex is told its writable roots explicitly, and they are exactly those three. | +| `network_access=False` | No sockets. The model reads production logs with nowhere to send them. Only the `workspace_write` sandbox honours this flag — `read_only` and `full_access` ignore it, and `full_access` + `network_access=False` now raises rather than pretending to isolate. | +| `approval_mode="deny_all"` | Escalation requests out of the sandbox are refused. **Never use `"auto_review"` in an example or in production**: it is not a review step. The Codex SDK's built-in handler accepts every escalation and cannot be replaced, so it is full auto-approval. | +| `RunConfig(max_llm_calls=40)` | A hard ceiling on a self-directed loop. Under `codex` the budget is charged *before* each backend call, so it binds exactly rather than one call late. | +| `max_tool_iterations=12` | ADK tool round-trips allowed for the **whole turn** (not per backend request). Four fetches plus a ticket fits with room to spare. | +| `tool_timeout_seconds=60.0` | A wedged ADK tool cannot hang the turn. | +| `outbox/` outside the workspace | Egress is a code path you own, not a file the model can drop somewhere. | +| no `workspace_root` / `reuse_workspace` | One workspace per `(app, user, session, agent)`, reaped when idle. Two incidents investigated at once cannot read each other's files, and yesterday's run cannot leave data in today's sandbox. | + +This is checkable, not aspirational. Asking the agent to try it, from inside +its own sandbox: + +``` +touch ./inside-ok -> exit 0 +touch ../outbox/ESCAPED -> touch: ../outbox/ESCAPED: Operation not permitted +touch ../ESCAPED2 -> touch: ../ESCAPED2: Operation not permitted +``` + +**One thing `workspace_write` does not do: it does not restrict *reads*.** The +sandbox permits reading the filesystem and only constrains writes. Nothing can +leave — no network, and the only egress is a tool whose payload you inspect — +but if the host holds secrets you would rather a model never read, run the +agent in a container or on a dedicated box. + +## The incident, so you can check the answer + +Everything is generated deterministically by `ops_backend.py` from a fixed +seed, so the same incident appears on every machine. Ground truth for +**2026-08-24 UTC**: + +- **Root cause.** `checkout-api 4.11.0` deployed at **14:07:55Z** raised worker + concurrency 8 → 32 while the DB connection pool stayed at 20. A new error + signature, `db.pool acquire timeout after 5000ms`, is **zero all day until + 14:11:02Z** and then grows to 640 occurrences. `db.pool.in_use` climbs from + ~6 and pegs at exactly `db.pool.size` = 20; p99 latency goes 180 ms → 5,200 + ms and 5xx/min goes 0.5 → 9.0, both sustained to the end of the window. +- **Decoy 1 — the loudest error.** `payments.gateway timeout` is the single + most frequent ERROR (951, more than the real signature's 640), and it runs at + a flat 35–45/hour all day and all of the previous week. Ranking signatures by + count picks it, and it is chronic noise. +- **Decoy 2 — the other deploy.** `notify-worker 2.3.1` shipped at 13:52:40Z, + fifteen minutes before the real one, and produced an immediate burst of 124 + `notify.webhook retry exhausted` errors. The burst stops on its own after six + minutes. "Blame the most recent deploy before the errors" picks this. +- **Decoy 3 — the biggest number on the dashboard.** `notify.queue_depth` + spikes from 12 to **900** between 13:52 and 14:25, peaking *before* the real + onset and recovering fully. It dwarfs every other metric movement. +- **The ruled-out hypothesis.** `checkout.rps` is identical on both days, so + the incident is not load-driven. +- **A single grep is not enough** by construction: the answer needs per-hour + counts per signature, a change point, a pivot of a long-format CSV, and a + join against deploy records — whose timestamps come from a CI system in + **UTC+08:00** while the logs are UTC and the metrics are epoch seconds. And + the log stream is not uniform: ~14% of lines are plaintext sidecar output, so + `json.loads` on every line raises. + +The follow-up turn asks about **2026-08-17**, the same weekday a week earlier. +The correct answer is *no*: zero pool timeouts, flat metrics, same chronic +payment noise. + +## What actually happened when we ran it + +Ark `deepseek-v4-pro-260425`, four runs of the full two-turn session. + +**Turn 1 found the intended root cause in all four.** 4 ADK tool calls and +10–12 sandboxed commands, 3.5–4 minutes. The shape was the same every time: +read the runbook skill, fetch all three sources, `head` the log and the CSV — +which is how it discovered the stream is not uniformly JSON and wrote a parser +that skips non-JSON lines — then write and run a series of separate programs +under `analysis/`, each answering the question the previous one raised: a +per-hour signature histogram, then a change-point locator, then a metrics +aggregation, then a pre/post-deploy comparison. + +Every ticket named `checkout-api 4.11.0`, concurrency 8→32 against a pool of +20, converted `22:07:55 +0800` to `14:07:55Z` correctly, and explicitly ruled +out both decoy deploys and the chronic payment timeouts in the evidence list. +One evidence item, verbatim: + +> `db.pool.in_use: before avg=6.0 (max=8.0), after avg=19.2 (max=20.0).` +> `Saturated (in_use >= pool size of 20) for 531/592 minutes (89.7%) after deploy.` + +**Turn 2 was where it went wrong, twice, for one concrete reason.** Two of the +four runs answered cleanly in 1–8 sandboxed commands, reusing the analysis from +turn 1 instead of re-deriving it. The other two got stuck: the scripts written +in turn 1 had the 08-24 filename baked in, so instead of parameterizing them +the model started renaming data files to fit — `mv`, run, `rm`, `mv` back — and +looped until `RunConfig(max_llm_calls=40)` cut the turn off. That is the +ceiling doing its job, not a bug; `main.py` catches it and prints +`[budget] the turn hit the 40-call ceiling`. But it is a fair warning that a +self-directed loop needs a budget, not just good intentions. + +That failure is also what earned the skill its last paragraph. Adding one line +to the runbook — *take the input path as `sys.argv[1]`, never hardcode it* — +changed turn 2 to a single clean invocation: + +``` +[sandbox 1] python3 analysis/error_signatures.py logs/checkout-api-prod_20260817T0000_20260818T0000.log +``` + +Honest caveat about iteration: **turn 1 never produced a failing command.** Its +iteration was refinement-driven — each program written to answer what the last +one turned up — not crash-driven, because the model dodged the mixed-format +trap by looking at the file before parsing it, which is what you would want. +The failing commands we did see (`exit 1`, three of them) came from the turn-2 +rabbit hole. Expect variance; a weaker model will fail more, and you will see +it in the `[sandbox N] -> exit ...` lines `main.py` prints. + +## The skill + +`skills/incident-triage/SKILL.md` is your team's triage runbook — pull all +sources first, characterize signatures instead of ranking them, locate the +change point, correlate with deploys, confirm in the metrics, explain the +mechanism, file one ticket. Codex's *native* skill system drives it: VeADK +materializes the ADK `SkillToolset` into `$CODEX_HOME/skills/`, and Codex +discovers and loads it on its own (the first sandboxed command in our run was +Codex reading the SKILL.md). + +It deliberately encodes **method, not answers** — it never mentions the log +format, the timezone quirk, or any of the three decoys by name. That is what +makes it a runbook rather than a spoiler, and it is why the agent still has to +do the work. + +## Running it + +```bash +pip install "veadk-python[codex]" # openai-codex + the bundled Codex CLI +cd examples/codex_ops_assistant +cp .env.example .env # add your Ark key +python main.py +``` + +macOS or Linux only: the sandbox is seatbelt / landlock+seccomp. Pick a model +with solid tool-calling and code-writing ability — this agent debugs its own +scripts — and read the two Ark gotchas below before swapping the model. + +Afterwards, look at what left the sandbox: + +```bash +cat outbox/INC-*.json # everything that left the sandbox +``` + +The workspace itself — the fetched data and the programs the model wrote under +`analysis/` — is printed by `main.py` just before it exits, because the +per-session workspace lives under a temporary root the runtime removes on +process exit. Nothing to clean up between runs, and no way for last week's files +to still be in the sandbox while the model investigates today's incident. Pin +`workspace_root` + `reuse_workspace` if you would rather keep the directory +around and inspect it later. + +## Two Ark gotchas worth knowing before you port an agent + +**1. Prompt caching cannot ride on a Codex turn.** VeADK enables Ark prompt +caching by default (`extra_body={"caching": {"type": "enabled"}}`), and Codex +always sends a top-level `instructions` field. Ark refuses the combination: + +``` +InvalidParameter: The parameter `instructions` specified in the request are not +valid: caching is not supported for instructions. +``` + +The codex shim strips `caching` and `expire_at` out of `extra_body` for you, so +you do not have to do anything — but the codex runtime forwards the *rest* of +`model_extra_config` verbatim (unlike `piagent`, which drops it entirely), so a +body key your backend dislikes will 400 every turn and the agent will answer +with an empty string. If you ever see the error above, set +`MODEL_AGENT_CACHING=disabled` or pass +`model_extra_config={"extra_body": {"caching": {"type": "disabled"}}}`. + +**2. Not every Ark model accepts Codex's conversation.** Codex replays +`reasoning` items in the request `input`. Models that do not support them abort +the turn part-way through, after the first tool round: + +``` +InvalidParameter: The parameter `input[3].reasoning` ... Item reasoning is not +supported for model: doubao-seed-1-6, version: 250615 +``` + +We saw exactly that with `doubao-seed-1-6-250615`: the agent fetched two files, +ran one `grep`, and then died mid-investigation. `deepseek-v4-pro-260425` (the +default here) accepts them and completes. Try a short turn on a new model +before pointing it at real work. + +## Files + +| File | What it is | +| --- | --- | +| `main.py` | The agent, the sandbox configuration, and a two-turn session that narrates every sandboxed command. | +| `ops_tools.py` | The four ADK tools: three that write files into the workspace and return receipts, one that files a ticket to `outbox/`. | +| `ops_backend.py` | The simulated internal observability system. Deterministic; this is where the incident and the decoys are defined. | +| `skills/incident-triage/SKILL.md` | The triage runbook, driven by Codex's native skill system. | diff --git a/examples/codex_ops_assistant/README.zh.md b/examples/codex_ops_assistant/README.zh.md new file mode 100644 index 000000000..a98b4bdf9 --- /dev/null +++ b/examples/codex_ops_assistant/README.zh.md @@ -0,0 +1,303 @@ +# 运维故障定位助手（`runtime="codex"`） + +> English version: [README.md](./README.md) + +**模型可以读取一整天的生产日志，却在物理上无法把它们传出去。** 它运行在操作系统级 +沙箱里，`network_access=False`——没有任何 socket 可以把数据送出；它也只能写自己的 +临时目录。通往外部世界的唯一通道是 `file_incident_ticket`，一个你自己写的 ADK +工具，它的参数你可以记录、校验、截断。 + +这就是对"我敢让大模型碰我的日志吗"这个问题的回答，也是本示例存在的理由。 + +它做什么：一个 on-call 智能体去调查一起它从未见过的故障。ADK 工具从内部系统拉取 +原始日志、指标序列和发布记录，作为文件落进沙箱。随后 Codex 在沙箱里自己写 shell 和 +Python 去 grep、聚合、做时间关联——写好几个程序，边看数据形态边改——最后开出一张 +工单，给出根因和支撑它的具体数字。 + +``` + ┌── ADK 工具（你的代码，你的进程）────────┐ + 内部 │ fetch_application_logs │ workspace/ + 系统 ───► │ fetch_service_metrics 写文件 ───────┼───► logs/*.log + │ fetch_deploy_history │ metrics/*.csv + └─────────────────────────────────────────┘ deploys/*.json + │ + ┌────────────────────────────────────────────┘ + ▼ + Codex 在操作系统沙箱内：没有网络，也不能写工作区以外的任何位置。 + 它写 analysis/*.py，运行它们，读输出，再写下一个。 + │ + ▼ file_incident_ticket(...) ← 唯一的出口 + outbox/INC-....json （在工作区之外，沙箱够不到） +``` + +## 为什么这个任务适合 codex 而不是 adk + +先说实话：对绝大多数智能体来说，`runtime="codex"` 严格劣于默认的 `runtime="adk"`。 +它每一轮都要拉起一个子进程，把整段对话重新序列化进提示词，还拒绝一大批 ADK 能力。 +只有当智能体真正的工作就是**运行它刚写出来的代码**时，这些代价才值得。临时性的日志 +分析正是这种情况： + +- **你没法为每个问题预先造一个工具。** "14:00 之后哪个错误签名的频率变了？""连接池 + 是在延迟上升之前还是之后打满的？""这个模式一周前有没有出现？"——对一个日志文件 + 有用的聚合方式是无穷的，每次新故障都会问出新的一个。按问题造工具是没有尽头的 + 跑步机，解释器不是。 +- **数据放不进上下文，而且模型不会数数。** 一天就是 7,494 行日志（1.4 MB）加 8,640 + 行指标。就算你愿意付钱把它们贴进去，"951 次支付超时 vs 640 次连接池超时"也应该 + 交给 `collections.Counter`，而不是靠模型阅读。执行代码给出的是精确数字，摘要式 + 提示词给出的是听起来合理的数字。 +- **做对需要好几轮。** 在我们的实测里，智能体先看了日志文件的头部，发现它并非全是 + JSON，然后写了一个按小时统计错误签名的直方图，接着是变化点定位脚本、指标聚合 + 脚本、连接池饱和度校验脚本——每一个都基于上一个的结论。这个循环正是 Codex 的 + 原生工作方式：一次模型轮次里跑很多条命令，程序输出留在它自己的循环里，而不是 + 一个字节一个字节地穿过你的事件流。 +- **沙箱才让这件事变得可以接受。** 让模型对着生产日志写并执行任意代码，只有在这些 + 代码碰不到网络和磁盘其余部分时才是理智的。这是运行时的属性，不是一句 prompt 能 + 承诺的东西。 + +最接近的替代方案是 `runtime="adk"` 配 ADK 的 `code_executor`。诚实地比较： +`UnsafeLocalCodeExecutor` 在你自己的进程里执行模型写的代码；`ContainerCodeExecutor` +需要你运维一个 Docker daemon；`VertexAiCodeExecutor` 是 Google 的托管服务。codex +运行时给你的是本机上的操作系统级沙箱（macOS seatbelt / Linux landlock+seccomp）、 +一个贯穿整个会话的工作目录，以及一次模型轮次内跑很多条命令的智能体循环，而不是 +一次 LLM 调用一个代码块。 + +### 什么时候**不要**用这个运行时 + +- **问题是固定的。** 如果 on-call 永远只问"昨天有多少 5xx"，那就写那个工具。更便宜、 + 更快、确定性更好、也可测试。 +- **你需要这个运行时替换掉的 ADK 能力。** `veadk/runtime/compat.py` 会直接拒绝 + `sub_agents`、`model=`（请用 `model_name=`）、`output_schema`、`planner`、 + `code_executor`、`include_contents="none"` 和 `enable_supervisor`，并会警告 + `knowledgebase`、`example_store`、模型 fallback 链、按 LLM 调用的 callback 和 + 按调用的 tracing span 都会被丢弃。移植智能体之前请先读那个文件。 +- **单轮延迟或成本是约束。** 每一轮都会拉起一个 Codex 子进程，而沙箱循环会在一轮内 + 发起多次后端调用。 +- **数据绝对不能落到这台机器上。** 工作区是运行智能体那台机器上的真实目录。 + +## 让 ADK 工具在这里正确工作的那条规则 + +**工作区是数据平面，工具的参数和返回值是控制平面。** + +`runtime="codex"` 下的 ADK 工具并不在沙箱里执行。VeADK 的 shim 在你的进程里执行它， +并把它返回的任何东西作为函数调用结果粘进模型的上下文。所以一个返回 7,494 行日志的 +工具并不是"把日志给了模型"——它会毁掉这一轮，而且模型之后还是得去 grep 那段文本。 + +因此这里每个 `fetch_*` 工具都往工作区写一个文件，然后返回一张**回执**： + +```python +{"status": "ok", + "path": "logs/checkout-api-prod_20260824T0000_20260825T0000.log", + "lines": 7494, "bytes": 1396879, + "note": "one event per line, chronological; content not returned"} +``` + +大约 40 个 token，而不是 40 万个。模型拿到指针，沙箱拿到字节。这是把 ADK 工具和这个 +运行时结合时最重要的一件事，两个方向都成立： + +- **工具通过写文件把数据交进去**，返回路径，外加足够的形态信息（行数、列名、单位） + 供模型规划。 +- **模型通过工具参数把结论交出来**——参数很小、有结构，而且校验权在你手里： + `file_incident_ticket` 会限制字段长度、检查 severity 枚举，并为每张工单打印一行 + 审计日志。 + +### 工具往哪里写 + +ADK 工具每次被调用时，自己问一遍本轮的工作目录在哪： + +```python +from veadk.runtime.codex import current_workspace + +def fetch_application_logs(stream: str, start_time: str, end_time: str) -> dict: + workspace = current_workspace() # 本轮的工作目录，或者 None + if workspace is None: # 不在 codex 轮次里——返回错误，而不是猜 + return {"status": "error", "message": "no sandbox working directory"} + ... +``` + +这个值由 runtime 在每次工具调用前后绑定，所以一个进程里同时跑着多个会话时它依然是对的。 +因此这个示例把 `workspace_root` 和 `reuse_workspace` 都留空：每个 +`(app, user, session, agent)` 各得一个目录——这正是一个真实的 on-call 服务想要的， +同时排查的两起故障不该共用一个临时目录——而它照样能跨同一会话的多轮存活， +第二轮复用的就是这一点。 + +不在 codex 轮次里时（换了 runtime、被 `AgentTool` 调用、单元测试）， +`current_workspace()` 返回 `None` 而不是抛异常；这里的工具会把它转成一条模型能处理的普通错误结果。 + +钉死目录是反过来的取舍，而且是**单租户**下的取舍：`workspace_root` 加 +`reuse_workspace=True` 给你一个可预测、进程退出后还能 `ls` 的目录， +代价是所有会话共用它。 + +## 安全配置，逐项拆解 + +```python +codex_runtime_config=CodexRuntimeConfig( + sandbox="workspace_write", + network_access=False, + approval_mode="deny_all", + max_tool_iterations=12, + tool_timeout_seconds=60.0, +) +... +run_config=RunConfig(max_llm_calls=40) +``` + +| 配置项 | 它阻止了什么 | +| --- | --- | +| `sandbox="workspace_write"` | 模型写的代码**只能**在工作区（以及临时目录）里创建和覆盖文件。它碰不到你的源码树、你的 dotfiles，也碰不到 `outbox/`。Codex 会被明确告知它的可写根目录，而那恰好就是这三个。 | +| `network_access=False` | 没有 socket。模型读得到生产日志，却无处可送。只有 `workspace_write` 沙箱会遵守这个开关——`read_only` 和 `full_access` 会忽略它，而 `full_access` + `network_access=False` 现在会直接抛错，而不是假装做了隔离。 | +| `approval_mode="deny_all"` | 拒绝一切越出沙箱的提权请求。**永远不要在示例或生产里用 `"auto_review"`**：它不是审核步骤。Codex SDK 内置的处理器会接受每一个提权请求且无法替换，所以它等于全自动放行。 | +| `RunConfig(max_llm_calls=40)` | 给自驱循环设一个硬上限。在 `codex` 下，预算是在每次后端调用**之前**扣的，所以卡点是精确的，不会晚一次。 | +| `max_tool_iterations=12` | **整轮**允许的 ADK 工具往返次数（不是每次后端请求）。四次抓取加一次开单绰绰有余。 | +| `tool_timeout_seconds=60.0` | 卡住的 ADK 工具不会把整轮挂死。 | +| `outbox/` 在工作区之外 | 出口是一条你拥有的代码路径，而不是模型可以随手丢文件的地方。 | +| 不设 `workspace_root` / `reuse_workspace` | 每个 `(app, user, session, agent)` 一个工作区，空闲后被回收。同时排查的两起故障读不到彼此的文件，昨天那次运行也不会把数据留在今天的沙箱里。 | + +这是可以验证的，不是口号。让智能体在它自己的沙箱里试一下： + +``` +touch ./inside-ok -> exit 0 +touch ../outbox/ESCAPED -> touch: ../outbox/ESCAPED: Operation not permitted +touch ../ESCAPED2 -> touch: ../ESCAPED2: Operation not permitted +``` + +**有一件事 `workspace_write` 不做：它不限制*读取*。** 这个沙箱允许读取文件系统， +只约束写入。数据出不去——没有网络，唯一的出口是一个你能检查其载荷的工具——但如果 +这台机器上有你根本不希望模型读到的密钥，请把智能体放进容器或专用机器里跑。 + +## 这起故障的标准答案，方便你核对 + +一切都由 `ops_backend.py` 用固定随机种子确定性生成，所以每台机器上出现的都是同一起 +故障。**2026-08-24 UTC** 的标准答案： + +- **根因。** `checkout-api 4.11.0` 于 **14:07:55Z** 发布，把 worker 并发从 8 提到 + 32，而数据库连接池仍然是 20。一个新的错误签名 + `db.pool acquire timeout after 5000ms` 在**当天 14:11:02Z 之前始终为零**，之后 + 一路涨到 640 条。`db.pool.in_use` 从约 6 爬升并恰好卡死在 `db.pool.size` = 20； + p99 延迟从 180 ms 涨到 5,200 ms，5xx/分钟从 0.5 涨到 9.0，且都持续到窗口末尾。 +- **诱饵一：最吵的错误。** `payments.gateway timeout` 是数量最多的 ERROR（951 条， + 比真正的签名 640 条还多），而它整天、以及整个上一周都稳定在 35–45 条/小时。按 + 数量排序会选中它，而它只是长期噪声。 +- **诱饵二：另一次发布。** `notify-worker 2.3.1` 在 13:52:40Z 上线，比真正的那次早 + 十五分钟，并立刻带来 124 条 `notify.webhook retry exhausted` 错误。这波爆发六分钟 + 后自己停了。"怪罪错误之前最近的那次发布"会选中它。 +- **诱饵三：仪表盘上最大的那个数。** `notify.queue_depth` 在 13:52 到 14:25 之间从 + 12 冲到 **900**，峰值出现在真正的起点*之前*，之后完全恢复。它的幅度碾压其他任何 + 指标变化。 +- **应当被排除的假设。** `checkout.rps` 两天完全一致，所以这不是流量驱动的故障。 +- **一次 grep 一定不够**，这是设计出来的：答案需要按小时按签名计数、需要找变化点、 + 需要对长格式 CSV 做透视、还需要和发布记录做关联——而发布记录的时间戳来自 + **UTC+08:00** 的 CI 系统，日志是 UTC，指标是 epoch 秒。此外日志流并不统一：约 + 14% 的行是 sidecar 的纯文本输出，对每一行做 `json.loads` 会抛异常。 + +追问的第二轮问的是 **2026-08-17**，即一周前的同一个星期几。正确答案是*没有*：零条 +连接池超时，指标平稳，只有同样的长期支付噪声。 + +## 我们实际跑出来的结果 + +Ark `deepseek-v4-pro-260425`，完整的两轮会话，跑了四次。 + +**第一轮四次都找到了预期的根因。** 4 次 ADK 工具调用、10–12 条沙箱命令、3.5–4 分钟。 +每次的形态一致：先读 runbook skill，拉取三个数据源，`head` 日志和 CSV——正是这一步 +让它发现日志流并非全是 JSON，并写出会跳过非 JSON 行的解析器——然后在 `analysis/` 下 +写并运行一系列独立程序，每一个都在回答上一个提出的问题：按小时的签名直方图、变化点 +定位、指标聚合、发布前后对比。 + +每张工单都点名了 `checkout-api 4.11.0`、并发 8→32 对上大小为 20 的连接池，都正确地 +把 `22:07:55 +0800` 换算成了 `14:07:55Z`，并在证据里明确排除了两个诱饵发布和长期 +存在的支付超时。原文摘录一条证据： + +> `db.pool.in_use: before avg=6.0 (max=8.0), after avg=19.2 (max=20.0).` +> `Saturated (in_use >= pool size of 20) for 531/592 minutes (89.7%) after deploy.` + +**出问题的是第二轮，两次，原因非常具体。** 四次里有两次干净地在 1–8 条沙箱命令内 +答完，复用了第一轮的分析而不是重新推导。另外两次卡住了：第一轮写的脚本把 08-24 的 +文件名写死了，模型没有把它参数化，而是开始重命名数据文件去迁就脚本——`mv`、运行、 +`rm`、再 `mv` 回来——一直循环到 `RunConfig(max_llm_calls=40)` 把这一轮切断。这是 +上限在正常工作而不是 bug；`main.py` 会捕获并打印 +`[budget] the turn hit the 40-call ceiling`。但这也是一个公道的提醒：自驱循环需要 +预算，光靠良好意愿不行。 + +这次失败也正是 skill 最后一段的由来。在 runbook 里加一行——*把输入路径作为 +`sys.argv[1]` 接收，绝不写死*——就把第二轮变成了一次干净的调用： + +``` +[sandbox 1] python3 analysis/error_signatures.py logs/checkout-api-prod_20260817T0000_20260818T0000.log +``` + +关于"迭代"的诚实说明：**第一轮从未产生过失败的命令。** 它的迭代是"逐步细化"型的 +（每个程序都是为了回答上一个程序带出的问题），而不是"崩溃后修复"型——模型在解析前 +先看了文件，从而避开了混合格式的坑，这本来也正是你希望它做的。我们确实看到的失败 +命令（三条 `exit 1`）来自第二轮的那个死胡同。请预期波动：能力更弱的模型会失败得更 +多，你会在 `main.py` 打印的 `[sandbox N] -> exit ...` 行里看到。 + +## 关于 skill + +`skills/incident-triage/SKILL.md` 就是你们团队的故障定位 runbook——先拉全部数据源、 +刻画签名而不是给签名排名、定位变化点、和发布记录关联、在指标里验证、解释机理、只开 +一张工单。驱动它的是 Codex 的**原生** skill 系统：VeADK 把 ADK 的 `SkillToolset` +物化到 `$CODEX_HOME/skills/`，Codex 自己发现并加载（我们那次运行的第一条沙箱命令就是 +Codex 在读这个 SKILL.md）。 + +它刻意只编码**方法，不编码答案**——从不提及日志格式、时区陷阱，也不点名任何一个 +诱饵。这才让它成为 runbook 而不是剧透，也正因如此智能体仍然得自己把活干完。 + +## 运行方式 + +```bash +pip install "veadk-python[codex]" # openai-codex + 自带的 Codex CLI +cd examples/codex_ops_assistant +cp .env.example .env # 填入你的方舟 API Key +python main.py +``` + +仅支持 macOS 或 Linux：沙箱是 seatbelt / landlock+seccomp。请选一个工具调用和写代码 +都过关的模型——这个智能体要给自己的脚本 debug——换模型之前请先读下面两个方舟坑。 + +跑完之后看看有什么离开了沙箱： + +```bash +cat outbox/INC-*.json # 所有离开沙箱的内容 +``` + +工作区本身——抓下来的数据，以及模型写在 `analysis/` 下的那些程序——由 `main.py` +在退出前打印出来：按会话隔离的工作区位于 runtime 自己的临时根目录下，进程退出时即被清理。 +所以两次运行之间没有什么要清理的，也不会出现「模型在排查今天的故障，沙箱里却还躺着上周的文件」。 +如果你更想把目录留下来事后慢慢看，就把 `workspace_root` + `reuse_workspace` 钉死。 + +## 换模型前值得知道的两个方舟坑 + +**1. 提示词缓存搭不了 Codex 这班车。** VeADK 默认开启方舟提示词缓存 +（`extra_body={"caching": {"type": "enabled"}}`），而 Codex 总是会发送顶层的 +`instructions` 字段，方舟拒绝这个组合： + +``` +InvalidParameter: The parameter `instructions` specified in the request are not +valid: caching is not supported for instructions. +``` + +codex 的 shim 会替你把 `extra_body` 里的 `caching` 和 `expire_at` 剔掉，所以你不用 +做什么——但 codex 运行时会原样转发 `model_extra_config` 的**其余部分**（与 `piagent` +不同，后者是整个丢弃），所以一个后端不认识的 body 字段会让每一轮都 400，智能体只 +返回空字符串。如果你真的看到上面那个错误，请设 `MODEL_AGENT_CACHING=disabled`，或者 +传 `model_extra_config={"extra_body": {"caching": {"type": "disabled"}}}`。 + +**2. 不是每个方舟模型都能接受 Codex 的对话结构。** Codex 会在请求的 `input` 里回放 +`reasoning` 项。不支持它们的模型会在第一轮工具调用之后中途终止： + +``` +InvalidParameter: The parameter `input[3].reasoning` ... Item reasoning is not +supported for model: doubao-seed-1-6, version: 250615 +``` + +我们在 `doubao-seed-1-6-250615` 上就遇到了：智能体抓取了两个文件、跑了一条 `grep`， +然后在调查中途挂掉。`deepseek-v4-pro-260425`（这里的默认值）能接受并跑完。换新模型 +之前，先用一个短轮次试一下。 + +## 文件说明 + +| 文件 | 是什么 | +| --- | --- | +| `main.py` | 智能体、沙箱配置，以及一个会把每条沙箱命令都叙述出来的两轮会话。 | +| `ops_tools.py` | 四个 ADK 工具：三个往工作区写文件并返回回执，一个把工单写进 `outbox/`。 | +| `ops_backend.py` | 被模拟的内部可观测性系统。确定性生成；故障和诱饵都定义在这里。 | +| `skills/incident-triage/SKILL.md` | 故障定位 runbook，由 Codex 原生 skill 系统驱动。 | diff --git a/examples/codex_ops_assistant/main.py b/examples/codex_ops_assistant/main.py new file mode 100644 index 000000000..3d2eadfb0 --- /dev/null +++ b/examples/codex_ops_assistant/main.py @@ -0,0 +1,268 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An on-call triage agent that investigates an incident inside a sandbox. + +Demonstrates the case `runtime="codex"` exists for: a question nobody can +pre-build a tool for. ADK tools pull raw logs, metrics and deploy records out +of an internal system and drop them into Codex's workspace; Codex then writes +throwaway shell and Python *in an OS sandbox* to grep, aggregate and correlate +them, refining one program into the next as it learns the shape of the data; +finally one ADK tool files a ticket. + +The security posture is the demo, not boilerplate: + +- ``sandbox="workspace_write"`` — Codex may write only its own workspace. +- ``network_access=False`` — the model can read production logs and physically + cannot exfiltrate them. There is no socket to send them out of. +- ``approval_mode="deny_all"`` — escalations out of the sandbox are refused, + not auto-approved. +- ``RunConfig(max_llm_calls=...)`` — a hard ceiling on a self-directed loop. + +The only egress is ``file_incident_ticket``, an audited ADK tool that writes to +``outbox/`` — a directory outside the workspace that the sandbox cannot reach. + +Run: + cd examples/codex_ops_assistant && python main.py + +Requires: +- ``pip install "veadk-python[codex]"`` (openai-codex plus the bundled Codex + CLI binary). macOS or Linux: the sandbox is seatbelt / landlock+seccomp. +- Ark (or another OpenAI-compatible chat) credentials via + ``MODEL_AGENT_API_KEY`` / ``MODEL_AGENT_API_BASE`` / ``MODEL_AGENT_NAME``. +""" + +import asyncio +import os +from pathlib import Path + +from google.adk.agents import RunConfig +from google.adk.agents.invocation_context import LlmCallsLimitExceededError +from google.adk.skills import load_skill_from_dir +from google.adk.tools.skill_toolset import SkillToolset +from google.genai import types +from ops_tools import OPS_TOOLS, OUTBOX, last_seen_workspace + +from veadk import Agent, Runner +from veadk.memory.short_term_memory import ShortTermMemory +from veadk.runtime.codex import CodexRuntimeConfig + +_HERE = Path(__file__).resolve().parent +_SKILL_DIR = _HERE / "skills" / "incident-triage" + +SESSION_ID = "incident-2026-08-24" + +MAX_LLM_CALLS = 40 +"""Hard ceiling on backend calls per turn. The sandboxed loop is self-directed: +Codex issues one backend call per native tool round, and `max_tool_iterations` +bounds only the ADK tool round-trips, not those. This is the knob that stops a +model that starts repeating itself.""" + +INSTRUCTION = """\ +You are an on-call SRE assistant for the `checkout-api` service. You +investigate incidents by analyzing raw telemetry yourself. + +Your working directory is a sandbox. You may create, run and rewrite files +there — shell, awk, and Python are all available. You have NO network access, +and you cannot write anywhere outside this directory. + +How to get data: +- The `fetch_*` tools do NOT return data. They download a file into your + working directory and hand you back a receipt with its path and size. Read + and analyze those files with your own commands. +- The files are large. Never cat a whole log file; aggregate it. +- Only the Python standard library is installed. There is no pandas, no numpy, + no jq. Write plain Python, or use grep/awk/sort/uniq. + +How to report: +- `file_incident_ticket` is the only channel out of this machine. Nothing you + write to a file will be read by anyone. File exactly one ticket at the end, + with a named root cause and the concrete numbers that support it. + +Be rigorous. State what the data shows, and say so plainly if the evidence +does not support a conclusion. +""" + +FIRST_QUESTION = """\ +Checkout error rates and latency were elevated on 2026-08-24 (UTC) and we do +not know why. Investigate the full day and file a ticket with the root cause. +""" + +FOLLOW_UP_QUESTION = """\ +Now check whether the same pattern was already happening a week earlier, on +2026-08-17 (UTC). Fetch that day's logs, metrics and deploys, then run the +analysis scripts you already have in `analysis/` against them. Answer in one +short paragraph, and do not file a second ticket. +""" + + +def build_agent() -> Agent: + """Build the triage agent, sandbox settings and all.""" + triage_runbook = SkillToolset(skills=[load_skill_from_dir(str(_SKILL_DIR))]) + + return Agent( + name="ops_triage_agent", + description="Investigates checkout incidents from raw logs and metrics.", + instruction=INSTRUCTION, + runtime="codex", + # `model_name`, never `model=`: the codex runtime resolves the model by + # name and would silently ignore a model object. + model_name=os.getenv("MODEL_AGENT_NAME", "deepseek-v4-pro-260425"), + model_api_base=os.getenv( + "MODEL_AGENT_API_BASE", "https://ark.cn-beijing.volces.com/api/v3" + ), + model_api_key=os.getenv("MODEL_AGENT_API_KEY"), + tools=[*OPS_TOOLS, triage_runbook], + codex_runtime_config=CodexRuntimeConfig( + # Codex may write inside the workspace and nowhere else. Its own + # scripts, its scratch files and the fetched data all live here. + sandbox="workspace_write", + # The whole point: the model reads production logs with no way to + # send them anywhere. Only honoured by `workspace_write`. + network_access=False, + # Refuse every escalation Codex asks for. Never use "auto_review" + # in production — it auto-approves, it does not review. + approval_mode="deny_all", + # `workspace_root` and `reuse_workspace` are left unset, which is + # what a real on-call service wants: one reaped directory per + # (app, user, session, agent), still shared by the turns of that + # session, never shared between two incidents. The ADK tools find + # it per call with `current_workspace()` — see `ops_tools.py`. + # + # ADK tool round-trips allowed for the whole turn (not per model + # request). Four fetches plus a ticket fits comfortably. + max_tool_iterations=12, + tool_timeout_seconds=60.0, + reasoning_effort="medium", + ), + ) + + +def _summarize(value: object, limit: int = 110) -> str: + text = " ".join(str(value).split()) + return text if len(text) <= limit else text[: limit - 1] + "…" + + +async def run_turn(runner: Runner, prompt: str, *, title: str) -> dict: + """Run one turn, narrating what the sandbox does, and return a tally.""" + print(f"\n{'=' * 78}\n{title}\n{'=' * 78}\nUser: {_summarize(prompt, 300)}\n") + tally = { + "sandbox_commands": 0, + "adk_tool_calls": 0, + "failed_commands": 0, + "stopped_by_budget": False, + } + answer = "" + + try: + async for event in runner.run_async( + user_id=runner.user_id, + session_id=SESSION_ID, + new_message=types.Content(role="user", parts=[types.Part(text=prompt)]), + run_config=RunConfig(max_llm_calls=MAX_LLM_CALLS), + ): + for call in event.get_function_calls() or []: + if call.name == "exec_command": + tally["sandbox_commands"] += 1 + print( + f" [sandbox {tally['sandbox_commands']:>2}] " + f"{_summarize(call.args.get('command'))}" + ) + else: + tally["adk_tool_calls"] += 1 + print(f" [adk tool ] {call.name}({_summarize(call.args, 90)})") + + for response in event.get_function_responses() or []: + payload = ( + response.response if isinstance(response.response, dict) else {} + ) + if response.name == "exec_command": + if payload.get("exit_code") not in (0, None): + tally["failed_commands"] += 1 + print( + f" -> exit {payload.get('exit_code')} " + f"(the model has to fix this)" + ) + elif payload.get("status") == "error": + print( + f" -> error: {_summarize(payload.get('message'))}" + ) + elif "path" in payload: + size = ( + payload.get("lines") + or payload.get("rows") + or payload.get("records") + ) + print(f" -> {payload['path']} ({size} entries)") + + if event.partial or not event.content or not event.content.parts: + continue + for part in event.content.parts: + if part.text and not part.thought: + answer = part.text + except LlmCallsLimitExceededError: + # Not a bug: this is `RunConfig(max_llm_calls=...)` doing its job. The + # sandboxed loop is self-directed, and a model that starts repeating + # itself would otherwise run until the wall clock stopped it. + tally["stopped_by_budget"] = True + print( + f"\n [budget] the turn hit the {MAX_LLM_CALLS}-call ceiling and was " + "stopped. The workspace still holds everything it produced." + ) + + if answer: + print(f"\nAgent: {answer.strip()}\n") + print( + f" turn used {tally['sandbox_commands']} sandboxed commands " + f"({tally['failed_commands']} of them failed and were retried) and " + f"{tally['adk_tool_calls']} ADK tool calls" + ) + return tally + + +async def main() -> None: + # Nothing to clear before the run. This used to delete the pinned + # `workspace/` directory, which `reuse_workspace=True` would otherwise have + # carried over from the previous run — last week's logs and scripts sitting + # in the sandbox while the model investigates today's incident. The + # per-session workspace makes that impossible: the runtime creates it under + # a temporary root of its own, one per process, and removes it on exit. + runner = Runner(agent=build_agent(), short_term_memory=ShortTermMemory()) + await runner.short_term_memory.create_session( + app_name=runner.app_name, user_id=runner.user_id, session_id=SESSION_ID + ) + + await run_turn(runner, FIRST_QUESTION, title="Turn 1 — investigate the incident") + # Same session, same workspace: the fetched data and the scripts the model + # wrote in turn 1 are still on disk, so it does not start from zero. + await run_turn(runner, FOLLOW_UP_QUESTION, title="Turn 2 — was it new?") + + # Listed here rather than left for you to `ls`: the runtime removes its + # per-session workspaces when the process exits. Pin `workspace_root` + + # `reuse_workspace` for a single-tenant run you want to inspect afterwards. + workspace = last_seen_workspace() + if workspace is None: + print("\nWorkspace: no tool ran, so nothing reported a workspace.") + else: + print(f"\nWorkspace (what the sandbox wrote): {workspace}") + for path in sorted(workspace.rglob("*")): + if path.is_file(): + print(f" {path.relative_to(workspace)} ({path.stat().st_size} bytes)") + print(f"\nOutbox (what left the sandbox): {OUTBOX}") + for path in sorted(OUTBOX.glob("*.json")): + print(f" {path.name}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/codex_ops_assistant/ops_backend.py b/examples/codex_ops_assistant/ops_backend.py new file mode 100644 index 000000000..8facf4e5c --- /dev/null +++ b/examples/codex_ops_assistant/ops_backend.py @@ -0,0 +1,501 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A simulated internal observability backend for the ops-triage example. + +Stands in for the log store, metrics store and deploy tracker a real on-call +engineer would query. Everything is generated from a fixed seed, so the same +incident is reproducible on every machine and the intended root cause can be +verified by hand (see ``README.md``). + +The data is deliberately awkward, the way production data is: + +- The log stream mixes **JSON lines** (the application) with **plaintext + lines** (an Envoy sidecar), so a parser that assumes one format crashes. +- Log timestamps are UTC ISO-8601, metric timestamps are **epoch seconds**, + and deploy timestamps come from a CI system in **UTC+08:00**. Correlating + the three requires normalizing all of them. +- The loudest error signature is chronic background noise, and the largest + metric spike belongs to an unrelated service. + +Nothing here is VeADK-specific; it is just the "internal system" the ADK tools +in ``ops_tools.py`` read from. +""" + +from __future__ import annotations + +import json +import math +import random +import zlib +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Iterator + +UTC = timezone.utc +CI_TZ = timezone(timedelta(hours=8)) +"""The CI system stamps deploys in UTC+08:00, not UTC. On purpose.""" + +SERVICE = "checkout-api" +LOG_STREAM = "checkout-api-prod" + +RETENTION_START = datetime(2026, 8, 10, tzinfo=UTC) +RETENTION_END = datetime(2026, 8, 26, tzinfo=UTC) + +INCIDENT_DAY = date(2026, 8, 24) +BASELINE_DAY = date(2026, 8, 17) +"""The same weekday, one week earlier: the "is this new?" comparison window.""" + +#: The deploy that actually breaks things, and the first minute it shows up in +#: the logs. Kept as module constants so the README's ground truth and the +#: generator cannot drift apart. +REGRESSION_DEPLOY_AT = datetime(2026, 8, 24, 14, 7, 55, tzinfo=UTC) +REGRESSION_ONSET_AT = datetime(2026, 8, 24, 14, 9, 0, tzinfo=UTC) +DB_POOL_SIZE = 20 + +_GATEWAYS = ("unionpay", "alipay", "visa-intl") +_COUPONS = ("coupon expired", "coupon not applicable", "quantity above limit") + + +# -------------------------------------------------------------------------- +# line formatting +# -------------------------------------------------------------------------- + + +def _iso(ts: datetime) -> str: + return ( + ts.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.") + + f"{ts.microsecond // 1000:03d}Z" + ) + + +def _trace(rng: random.Random) -> str: + return f"{rng.getrandbits(64):016x}" + + +def _json_line( + ts: datetime, level: str, service: str, component: str, msg: str, **fields +) -> str: + record = { + "ts": _iso(ts), + "level": level, + "service": service, + "component": component, + "msg": msg, + } + record.update(fields) + return json.dumps(record, separators=(",", ":")) + + +def _sidecar_line(ts: datetime, body: str) -> str: + """Plaintext sidecar line: a different format *and* a different clock.""" + stamp = ts.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S +0000") + return f"{stamp} [envoy-sidecar] {body}" + + +# -------------------------------------------------------------------------- +# log generation +# -------------------------------------------------------------------------- + + +def _pool_error_rate(hour: int) -> int: + """Errors/hour for the connection-pool leak, once it starts. + + Ramps up over the afternoon but stays *below* the chronic payment-gateway + noise in total volume, so ranking signatures by count alone points at the + wrong thing. + """ + ramp = { + 14: 2, + 15: 12, + 16: 28, + 17: 47, + 18: 66, + 19: 82, + 20: 95, + 21: 104, + 22: 110, + 23: 94, + } + return ramp.get(hour, 0) + + +def _generate_log_lines(day: date, rng: random.Random) -> list[tuple[datetime, str]]: + start = datetime(day.year, day.month, day.day, tzinfo=UTC) + incident = day == INCIDENT_DAY + out: list[tuple[datetime, str]] = [] + + def at(hour: int) -> datetime: + return start + timedelta(hours=hour, seconds=rng.uniform(0, 3600)) + + for hour in range(24): + # Successful checkouts: the bulk of the stream, and the reason a + # case-insensitive grep for "error" is useless here. + for _ in range(rng.randint(95, 115)): + ts = at(hour) + out.append( + ( + ts, + _json_line( + ts, + "INFO", + SERVICE, + "http", + "checkout completed", + trace_id=_trace(rng), + status=200, + duration_ms=rng.randint(60, 320), + ), + ) + ) + # Chronic validation warnings. + for _ in range(rng.randint(80, 100)): + ts = at(hour) + out.append( + ( + ts, + _json_line( + ts, + "WARN", + SERVICE, + "cart.validation", + f"line item rejected: {rng.choice(_COUPONS)}", + trace_id=_trace(rng), + cart_id=f"cart_{rng.getrandbits(32):08x}", + ), + ) + ) + # Decoy #1: the loudest ERROR in the file, at a flat rate all day and + # all of last week too. + for _ in range(rng.randint(35, 45)): + ts = at(hour) + out.append( + ( + ts, + _json_line( + ts, + "ERROR", + "payments-proxy", + "payments.gateway", + "gateway timeout after 3000ms", + trace_id=_trace(rng), + gateway=rng.choice(_GATEWAYS), + attempt=rng.randint(1, 3), + ), + ) + ) + # Sidecar heartbeat: plaintext, so json.loads() on every line fails. + for minute in range(0, 60, 2): + ts = start + timedelta(hours=hour, minutes=minute) + out.append( + ( + ts, + _sidecar_line( + ts, + "health_check ok cluster=checkout-api upstream=10.4.2.17:8080", + ), + ) + ) + + if not incident: + continue + + # The real signature: absent before the deploy, growing after it. + for _ in range(_pool_error_rate(hour)): + ts = max( + at(hour), REGRESSION_ONSET_AT + timedelta(seconds=rng.uniform(0, 60)) + ) + if ts.hour != hour: + continue + out.append( + ( + ts, + _json_line( + ts, + "ERROR", + SERVICE, + "db.pool", + "acquire timeout after 5000ms", + trace_id=_trace(rng), + pool_size=DB_POOL_SIZE, + pool_in_use=DB_POOL_SIZE, + wait_ms=rng.randint(5000, 5400), + ), + ) + ) + # Downstream symptom, visible only in the plaintext lines. + if rng.random() < 0.55: + out.append( + ( + ts, + _sidecar_line( + ts, + "upstream_reset_before_response_started{connection_termination} " + f"cluster=checkout-api-db req_id={_trace(rng)}", + ), + ) + ) + + if incident: + # Decoy #2: a short, loud burst right after the *other* deploy of the + # afternoon, which then stops on its own. + burst_start = datetime(2026, 8, 24, 13, 52, 40, tzinfo=UTC) + for _ in range(124): + ts = burst_start + timedelta(seconds=rng.uniform(0, 360)) + out.append( + ( + ts, + _json_line( + ts, + "ERROR", + "notify-worker", + "notify.webhook", + "retry exhausted after 5 attempts", + trace_id=_trace(rng), + endpoint="https://hooks.internal/checkout", + ), + ) + ) + + out.sort(key=lambda item: item[0]) + return out + + +# -------------------------------------------------------------------------- +# metric generation +# -------------------------------------------------------------------------- + + +def _ramp(elapsed_min: float, span_min: float, lo: float, hi: float) -> float: + return lo + (hi - lo) * min(1.0, max(0.0, elapsed_min / span_min)) + + +def _generate_metric_rows( + day: date, rng: random.Random +) -> list[tuple[int, str, float]]: + start = datetime(day.year, day.month, day.day, tzinfo=UTC) + incident = day == INCIDENT_DAY + onset = datetime(2026, 8, 24, 14, 10, tzinfo=UTC) + notify_from = datetime(2026, 8, 24, 13, 52, tzinfo=UTC) + notify_peak = datetime(2026, 8, 24, 14, 5, tzinfo=UTC) + notify_to = datetime(2026, 8, 24, 14, 25, tzinfo=UTC) + + rows: list[tuple[int, str, float]] = [] + for minute in range(24 * 60): + ts = start + timedelta(minutes=minute) + epoch = int(ts.timestamp()) + after = (ts - onset).total_seconds() / 60.0 if incident else -1.0 + + # Traffic is flat week over week: this rules out "we just got busier". + rps = ( + 40 + 25 * math.sin(2 * math.pi * (minute - 360) / 1440) + rng.uniform(-2, 2) + ) + p99 = 180 + rng.uniform(-25, 25) + errors = max(0.0, 0.4 + rng.uniform(-0.3, 0.5)) + in_use = min(12.0, max(2.0, 6 + rng.uniform(-2, 2))) + queue = 12 + rng.uniform(-4, 4) + + if after >= 0: + p99 = _ramp(after, 300, 180, 5200) + rng.uniform(-60, 60) + errors = _ramp(after, 300, 0.4, 9.0) + rng.uniform(-0.4, 0.4) + in_use = min( + float(DB_POOL_SIZE), _ramp(after, 60, 6, 20.6) + rng.uniform(-0.4, 0.2) + ) + if incident and notify_from <= ts <= notify_to: + # Decoy #3: by far the biggest number on any dashboard, belonging + # to a different service, and it starts *before* the real onset. + if ts <= notify_peak: + queue = _ramp((ts - notify_from).total_seconds() / 60, 13, 12, 900) + else: + queue = _ramp((notify_to - ts).total_seconds() / 60, 20, 12, 900) + + rows.extend( + [ + (epoch, "checkout.rps", round(rps, 1)), + (epoch, "http.p99_latency_ms", round(p99, 1)), + (epoch, "http.5xx_per_min", round(max(0.0, errors), 2)), + (epoch, "db.pool.in_use", round(in_use, 1)), + (epoch, "db.pool.size", float(DB_POOL_SIZE)), + (epoch, "notify.queue_depth", round(queue, 1)), + ] + ) + return rows + + +# -------------------------------------------------------------------------- +# deploy records +# -------------------------------------------------------------------------- + + +def _deploy( + when: datetime, service: str, version: str, author: str, changes: list[str] +) -> dict: + return { + "service": service, + "version": version, + "deployed_at": when.astimezone(CI_TZ).strftime("%Y-%m-%d %H:%M:%S %z"), + "deployed_by": author, + "pipeline": f"cicd-{when:%Y%m%d}-{zlib.crc32(version.encode()) % 9000 + 1000}", + "changes": changes, + } + + +def _generate_deploys(day: date) -> list[dict]: + if day == INCIDENT_DAY: + return [ + _deploy( + datetime(2026, 8, 24, 9, 31, 12, tzinfo=UTC), + SERVICE, + "4.10.3", + "wu.lei", + ["bump libcurl to 8.9.1", "fix typo in receipt email template"], + ), + _deploy( + datetime(2026, 8, 24, 13, 52, 40, tzinfo=UTC), + "notify-worker", + "2.3.1", + "chen.yu", + ["switch webhook retry to exponential backoff (max 5 attempts)"], + ), + _deploy( + REGRESSION_DEPLOY_AT, + SERVICE, + "4.11.0", + "zhao.min", + [ + "raise checkout worker concurrency 8 -> 32", + "cache tax tables in process", + "add /healthz readiness probe", + ], + ), + _deploy( + datetime(2026, 8, 24, 15, 40, 10, tzinfo=UTC), + "search-api", + "1.9.4", + "li.fang", + ["re-rank suggestions by conversion"], + ), + ] + if day == BASELINE_DAY: + return [ + _deploy( + datetime(2026, 8, 17, 10, 12, 4, tzinfo=UTC), + SERVICE, + "4.10.1", + "wu.lei", + ["add currency formatting for MYR"], + ) + ] + return [] + + +# -------------------------------------------------------------------------- +# store: materialize once, then query by time range +# -------------------------------------------------------------------------- + + +def _seed(day: date) -> int: + return int(day.strftime("%Y%m%d")) + + +def ensure_day(store: Path, day: date) -> None: + """Materialize one day of the simulated backend, if not already present.""" + logs = store / "logs" / f"{day.isoformat()}.log" + metrics = store / "metrics" / f"{day.isoformat()}.csv" + deploys = store / "deploys" / f"{day.isoformat()}.json" + if logs.exists() and metrics.exists() and deploys.exists(): + return + for path in (logs, metrics, deploys): + path.parent.mkdir(parents=True, exist_ok=True) + + rng = random.Random(_seed(day)) + logs.write_text( + "".join(f"{line}\n" for _, line in _generate_log_lines(day, rng)), + encoding="utf-8", + ) + rows = _generate_metric_rows(day, rng) + metrics.write_text( + "timestamp,metric,value\n" + "".join(f"{t},{m},{v}\n" for t, m, v in rows), + encoding="utf-8", + ) + deploys.write_text( + json.dumps(_generate_deploys(day), indent=2) + "\n", encoding="utf-8" + ) + + +def _days(start: datetime, end: datetime) -> Iterator[date]: + day = start.astimezone(UTC).date() + last = end.astimezone(UTC).date() + while day <= last: + yield day + day += timedelta(days=1) + + +def _line_time(line: str) -> datetime | None: + """Parse either log format's timestamp, or give up on a malformed line.""" + if line.startswith('{"ts":"'): + try: + return datetime.strptime(line[7:30], "%Y-%m-%dT%H:%M:%S.%f").replace( + tzinfo=UTC + ) + except ValueError: + return None + try: + return datetime.strptime(line[:25], "%Y-%m-%d %H:%M:%S %z") + except ValueError: + return None + + +def read_log_lines(store: Path, start: datetime, end: datetime) -> Iterator[str]: + """Yield raw log lines whose timestamp falls in ``[start, end)``.""" + for day in _days(start, end): + ensure_day(store, day) + path = store / "logs" / f"{day.isoformat()}.log" + with path.open(encoding="utf-8") as handle: + for line in handle: + stripped = line.rstrip("\n") + when = _line_time(stripped) + if when is not None and start <= when < end: + yield stripped + + +def read_metric_rows(store: Path, start: datetime, end: datetime) -> Iterator[str]: + """Yield raw ``timestamp,metric,value`` rows in ``[start, end)``.""" + lo, hi = int(start.timestamp()), int(end.timestamp()) + for day in _days(start, end): + ensure_day(store, day) + path = store / "metrics" / f"{day.isoformat()}.csv" + with path.open(encoding="utf-8") as handle: + next(handle, None) + for line in handle: + stripped = line.rstrip("\n") + if not stripped: + continue + epoch = int(stripped.split(",", 1)[0]) + if lo <= epoch < hi: + yield stripped + + +def read_deploys(store: Path, start: datetime, end: datetime) -> list[dict]: + """Return deploy records in ``[start, end)``, oldest first.""" + found: list[dict] = [] + for day in _days(start, end): + ensure_day(store, day) + path = store / "deploys" / f"{day.isoformat()}.json" + for record in json.loads(path.read_text(encoding="utf-8")): + when = datetime.strptime(record["deployed_at"], "%Y-%m-%d %H:%M:%S %z") + if start <= when < end: + found.append(record) + found.sort(key=lambda item: item["deployed_at"]) + return found diff --git a/examples/codex_ops_assistant/ops_tools.py b/examples/codex_ops_assistant/ops_tools.py new file mode 100644 index 000000000..f45866dc3 --- /dev/null +++ b/examples/codex_ops_assistant/ops_tools.py @@ -0,0 +1,364 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ADK tools for the Codex ops-triage example: the agent's only I/O. + +Two rules shape every tool here, and they are the point of the example: + +**The workspace is the data plane; tool results are the control plane.** +An ADK tool under ``runtime="codex"`` is executed by VeADK's shim, not inside +the sandbox, and whatever it returns is pasted into the model's context as the +function-call result. Returning 8,000 log lines would blow the turn's context +on data the model then still has to grep. So the ``fetch_*`` tools write a file +into Codex's workspace and return a **receipt** — path, size, shape — and the +model reads the file with its own sandboxed shell and Python. + +**The tools are the only egress.** Codex runs with ``network_access=False`` and +``sandbox="workspace_write"``, so it can read and rewrite everything in the +workspace and reach nothing else: no sockets, and no writes outside the +workspace. ``file_incident_ticket`` writes to ``outbox/``, which sits *outside* +the workspace and is therefore unreachable from inside the sandbox. Every byte +that leaves is a structured argument to this one function, on one audited code +path you own. + +Where the workspace is, the ``fetch_*`` tools ask per call: +:func:`veadk.runtime.codex.current_workspace` reports the directory of the turn +that is calling them. Nothing here pins ``workspace_root``, so every session +gets its own directory — what a real on-call service would want, since two +incidents investigated at once must not share a scratch directory. +""" + +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from veadk.runtime.codex import current_workspace + +from ops_backend import ( + LOG_STREAM, + RETENTION_END, + RETENTION_START, + SERVICE, + read_deploys, + read_log_lines, + read_metric_rows, +) + +_HERE = Path(__file__).resolve().parent + +OUTBOX = _HERE / "outbox" +"""Where filed tickets land. Outside the workspace, so the sandbox cannot.""" + +STORE = _HERE / "_store" +"""The simulated internal system's own storage. Also outside the workspace.""" + +MAX_RANGE = timedelta(days=3) +_MAX_TICKET_FIELD_CHARS = 4000 +_MAX_EVIDENCE_ITEMS = 20 + +_LAST_WORKSPACE: Path | None = None +"""The workspace the most recent tool call ran in — a *demo* affordance. + +``main.py`` lists the directory once the session is over, and this example does +not pin ``workspace_root``, so nothing outside a tool call knows the path. A +single-session script can remember it like this; a server handling several +incidents at once cannot, and does not need to — its tools are handed the right +directory on every call. +""" + + +def _workspace() -> Path | None: + """The workspace of the Codex turn calling this tool, or ``None``. + + Returns: + Path | None: Codex's working directory for this turn, or ``None`` when + the tool runs outside a codex turn (another runtime, an ``AgentTool``, + a unit test). + """ + global _LAST_WORKSPACE + workspace = current_workspace() + if workspace is None: + return None + _LAST_WORKSPACE = Path(workspace) + return _LAST_WORKSPACE + + +def last_seen_workspace() -> Path | None: + """The workspace observed by the last tool call. See :data:`_LAST_WORKSPACE`.""" + return _LAST_WORKSPACE + + +def _no_workspace_error() -> dict: + """The result to return when there is no workspace to download into. + + :func:`~veadk.runtime.codex.current_workspace` returns ``None`` rather than + raising when no codex turn is on the stack, so the tool answers in kind: an + error result the model can read beats an exception it cannot. + """ + return { + "status": "error", + "message": ( + "no sandbox working directory on this call, so nothing was " + "downloaded; these tools only work inside a codex turn." + ), + } + + +def _parse_time(value: str, label: str) -> datetime: + """Accept ISO-8601 with ``Z``, an offset, or none (treated as UTC).""" + text = (value or "").strip().replace("Z", "+00:00").replace(" ", "T", 1) + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise ValueError( + f"{label}={value!r} is not ISO-8601; use e.g. '2026-08-24T00:00:00Z'" + ) from exc + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _resolve_range(start_time: str, end_time: str) -> tuple[datetime, datetime]: + start = _parse_time(start_time, "start_time") + end = _parse_time(end_time, "end_time") + if end <= start: + raise ValueError("end_time must be after start_time") + if end - start > MAX_RANGE: + raise ValueError( + f"range is {end - start}; this backend serves at most {MAX_RANGE} " + "per call, so fetch one window at a time" + ) + if start < RETENTION_START or end > RETENTION_END: + raise ValueError( + "outside retention; logs and metrics exist from " + f"{RETENTION_START:%Y-%m-%dT%H:%M:%SZ} to " + f"{RETENTION_END:%Y-%m-%dT%H:%M:%SZ}" + ) + return start, end + + +def _slug(start: datetime, end: datetime) -> str: + return f"{start:%Y%m%dT%H%M}_{end:%Y%m%dT%H%M}" + + +def _write(workspace: Path, relative: str, text: str) -> tuple[str, int]: + """Write into this turn's workspace and return ``(relative path, bytes)``.""" + target = workspace / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + return relative, target.stat().st_size + + +def fetch_application_logs(stream: str, start_time: str, end_time: str) -> dict: + """Download raw application logs into your workspace as a file. + + The log content is NOT returned to you: it is far too large for the + conversation. Only a receipt comes back; read and analyze the file with + your own shell and Python. + + Args: + stream: Log stream to pull. The only stream you have access to is + 'checkout-api-prod' (the production checkout pod's combined + stdout). + start_time: Inclusive start of the window, ISO-8601 UTC, e.g. + '2026-08-24T00:00:00Z'. + end_time: Exclusive end of the window, ISO-8601 UTC. At most 3 days + after start_time. + + Returns: + dict: A receipt with the workspace-relative 'path', 'lines' and + 'bytes' on success, or 'status'='error' with a 'message' explaining + how to fix the call. + """ + workspace = _workspace() + if workspace is None: + return _no_workspace_error() + if stream != LOG_STREAM: + return { + "status": "error", + "message": f"unknown stream {stream!r}; available streams: [{LOG_STREAM!r}]", + } + try: + start, end = _resolve_range(start_time, end_time) + except ValueError as exc: + return {"status": "error", "message": str(exc)} + + lines = list(read_log_lines(STORE, start, end)) + path, size = _write( + workspace, + f"logs/{stream}_{_slug(start, end)}.log", + "".join(f"{line}\n" for line in lines), + ) + return { + "status": "ok", + "path": path, + "lines": len(lines), + "bytes": size, + "window_utc": [f"{start:%Y-%m-%dT%H:%M:%SZ}", f"{end:%Y-%m-%dT%H:%M:%SZ}"], + "note": "one event per line, chronological; content not returned", + } + + +def fetch_service_metrics(service: str, start_time: str, end_time: str) -> dict: + """Download a per-minute metric series into your workspace as a CSV file. + + As with the logs, the rows are NOT returned to you — only a receipt. + + Args: + service: Service to pull metrics for. Use 'checkout-api'. + start_time: Inclusive start of the window, ISO-8601 UTC. + end_time: Exclusive end of the window, ISO-8601 UTC. At most 3 days + after start_time. + + Returns: + dict: A receipt with the workspace-relative 'path', 'rows', 'columns' + and the metric names present, or 'status'='error' with a 'message'. + """ + workspace = _workspace() + if workspace is None: + return _no_workspace_error() + if service != SERVICE: + return { + "status": "error", + "message": f"unknown service {service!r}; available services: [{SERVICE!r}]", + } + try: + start, end = _resolve_range(start_time, end_time) + except ValueError as exc: + return {"status": "error", "message": str(exc)} + + rows = list(read_metric_rows(STORE, start, end)) + path, size = _write( + workspace, + f"metrics/{service}_{_slug(start, end)}.csv", + "timestamp,metric,value\n" + "".join(f"{row}\n" for row in rows), + ) + return { + "status": "ok", + "path": path, + "rows": len(rows), + "bytes": size, + "columns": ["timestamp", "metric", "value"], + "note": ( + "long format (one row per metric per minute); 'timestamp' is " + "epoch seconds in UTC" + ), + } + + +def fetch_deploy_history(start_time: str, end_time: str) -> dict: + """Download the deploy log for every service into your workspace as JSON. + + Args: + start_time: Inclusive start of the window, ISO-8601 UTC. + end_time: Exclusive end of the window, ISO-8601 UTC. At most 3 days + after start_time. + + Returns: + dict: A receipt with the workspace-relative 'path' and 'records', or + 'status'='error' with a 'message'. + """ + workspace = _workspace() + if workspace is None: + return _no_workspace_error() + try: + start, end = _resolve_range(start_time, end_time) + except ValueError as exc: + return {"status": "error", "message": str(exc)} + + records = read_deploys(STORE, start, end) + path, size = _write( + workspace, + f"deploys/deploys_{_slug(start, end)}.json", + json.dumps(records, indent=2) + "\n", + ) + return { + "status": "ok", + "path": path, + "records": len(records), + "bytes": size, + "note": "JSON array; 'deployed_at' is stamped by the CI system, not in UTC", + } + + +def file_incident_ticket( + title: str, + severity: str, + root_cause: str, + evidence: list[str], + recommended_action: str, +) -> dict: + """File an incident ticket. This is the ONLY way to report anything. + + Nothing you write in the workspace reaches a human. Call this exactly once, + when you can name a root cause and cite the evidence for it. + + Args: + title: One-line summary, e.g. 'checkout-api 5xx after '. + severity: One of 'sev1', 'sev2', 'sev3'. + root_cause: What actually broke and why, in a few sentences. Name the + specific change if you found one. + evidence: Concrete observations backing the root cause — counts, + timestamps, metric values. One string per observation, at most 20. + recommended_action: What on-call should do next. + + Returns: + dict: 'ticket_id' and the absolute 'path' the ticket was written to, + or 'status'='error' with a 'message'. + """ + severity = (severity or "").strip().lower() + if severity not in {"sev1", "sev2", "sev3"}: + return { + "status": "error", + "message": f"severity must be one of sev1/sev2/sev3, got {severity!r}", + } + if isinstance(evidence, str): + evidence = [evidence] + items = [str(item)[:_MAX_TICKET_FIELD_CHARS] for item in (evidence or [])] + if not items: + return {"status": "error", "message": "evidence must not be empty"} + + ticket_id = ( + f"INC-{datetime.now(timezone.utc):%Y%m%d}-{uuid.uuid4().hex[:6].upper()}" + ) + ticket = { + "ticket_id": ticket_id, + "filed_at": f"{datetime.now(timezone.utc):%Y-%m-%dT%H:%M:%SZ}", + "title": str(title)[:_MAX_TICKET_FIELD_CHARS], + "severity": severity, + "root_cause": str(root_cause)[:_MAX_TICKET_FIELD_CHARS], + "evidence": items[:_MAX_EVIDENCE_ITEMS], + "recommended_action": str(recommended_action)[:_MAX_TICKET_FIELD_CHARS], + } + OUTBOX.mkdir(parents=True, exist_ok=True) + target = OUTBOX / f"{ticket_id}.json" + target.write_text(json.dumps(ticket, indent=2, ensure_ascii=False) + "\n", "utf-8") + + # The audit point: one line per byte that leaves the sandbox. + print(f" [egress] ticket {ticket_id} ({severity}) -> {target}") + return { + "status": "ok", + "ticket_id": ticket_id, + "path": str(target), + "evidence_items_accepted": len(ticket["evidence"]), + } + + +OPS_TOOLS = [ + fetch_application_logs, + fetch_service_metrics, + fetch_deploy_history, + file_incident_ticket, +] diff --git a/examples/codex_ops_assistant/skills/incident-triage/SKILL.md b/examples/codex_ops_assistant/skills/incident-triage/SKILL.md new file mode 100644 index 000000000..cf3e46011 --- /dev/null +++ b/examples/codex_ops_assistant/skills/incident-triage/SKILL.md @@ -0,0 +1,67 @@ +--- +name: incident-triage +description: The on-call triage procedure for checkout-api. Use whenever you are asked to investigate elevated errors, elevated latency, or a suspected regression. +--- + +Follow this procedure. It exists because the obvious answer is usually wrong. + +## 1. Pull everything first + +Fetch logs, metrics **and** deploy history for the whole window before you +analyze any of them. A conclusion drawn from one source is a guess. + +## 2. Characterize signatures, do not rank them + +Never conclude from "which error is most frequent". High-volume errors are +usually chronic noise that was there yesterday too. + +For every distinct error signature, compute its count **per hour** across the +window. You are looking for a signature whose *rate changed* — ideally one that +was zero and then was not. A signature that is flat across the whole window is +background, however loud it is. + +## 3. Locate the change point + +For each signature that changed, find the timestamp of its first occurrence +after the change. That timestamp, not the start of the window, is the moment +you are explaining. + +## 4. Correlate with deploys + +Normalize every timestamp to UTC epoch seconds before comparing sources; they +do not all use the same format or the same clock. A deploy is a candidate only +if it precedes the change point by minutes, not hours. Where two deploys are +close together, the one that lines up is the one that lines up — check both. + +## 5. Confirm in the metrics + +The metric series is long format: one row per metric per minute. Aggregate it +per metric before drawing conclusions. + +- A cause produces a **sustained** change beginning at the change point. A + spike that recovers on its own is not your incident. +- Check request volume too. If traffic did not change, the incident is not + load-driven and you should stop looking for one. +- A resource metric sitting exactly at its configured limit is worth more than + any latency graph. + +## 6. Explain the mechanism + +Tie the candidate deploy's change list to what the metrics show. If you cannot +explain *how* that change produces *these* numbers, you have a correlation, not +a root cause — and you should say that in the ticket. + +## 7. File one ticket + +Exactly one, at the end. Every claim in `evidence` must be a number you +actually computed, with its timestamp. Rule out the hypotheses you rejected, +and say why. + +## Working notes + +Keep your analysis scripts in the workspace under `analysis/` with names that +say what they do. You will be asked to re-run them over a different window. + +Take the input file path as a command-line argument (`sys.argv[1]`). Never +hardcode it — a script with a filename baked in is not reusable, and renaming +data files to fit an old script wastes far more time than adding one argument. diff --git a/examples/codex_runtime_on_agentkit/README.md b/examples/codex_runtime_on_agentkit/README.md index 5d76a6413..c3c6db06d 100644 --- a/examples/codex_runtime_on_agentkit/README.md +++ b/examples/codex_runtime_on_agentkit/README.md @@ -6,6 +6,17 @@ A minimal deployable app whose agent runs on the **OpenAI Codex runtime** > 中文版见 [README.zh.md](./README.zh.md) +> **What this example is:** a **deployment reference**. It shows how to package +> and ship a `runtime="codex"` agent to AgentKit — the requirements pins, the +> bundled Codex binary, the `agentkit config` flags. The agent itself is a +> placeholder, and its one-shot Q&A is not a use case this runtime is good at: +> `runtime="adk"` answers that kind of question faster and cheaper. For what the +> runtime is actually *for*, see [`codex_data_analysis/`](../codex_data_analysis/) +> (a model that writes a script, runs it, reads the traceback and fixes it) and +> [`codex_ops_assistant/`](../codex_ops_assistant/) (the same loop over logs and +> metrics, under a no-network sandbox), or +> [when to use the codex runtime](../../docs/content/docs/framework/agent/runtime.en.mdx#when-to-use-the-codex-runtime). + ## What's inside ```text @@ -13,7 +24,7 @@ codex_runtime_on_agentkit/ ├── app.py # deploy entry point (ADK agent API server) ├── agents/ │ └── codex_agent/ # the agent — Agent(runtime="codex") -├── requirements.txt # veadk-python>=0.5.39 + openai-codex +├── requirements.txt # veadk-python + openai-codex + fastapi/uvicorn ├── .env.example └── .dockerignore ``` @@ -29,7 +40,13 @@ codex_runtime_on_agentkit/ - **`openai-codex` is not a veadk dependency**, so `requirements.txt` lists it explicitly. It pulls in `openai-codex-cli-bin`, which ships the Codex CLI binary as a **manylinux wheel** — no separate binary install in the Linux - build. + build. These pins mirror veadk-python's `[codex]` extra; the extra is not + used directly because uv only accepts a pre-release when its exact version is + pinned at the top level, not transitively through an extra. +- `fastapi` and `uvicorn` are listed too: `app.py` imports `uvicorn` directly + and the runtime's Responses→chat shim imports both at module level. They + resolve through google-adk today, but adk has been moving web deps behind + extras, so `[codex]` declares them explicitly and so does this file. > The codex runtime is included in `veadk-python` since **0.5.39** (on PyPI), so > the image installs everything from PyPI via the default @@ -46,7 +63,7 @@ cp .env.example .env ## 2. Run locally (optional) ```bash -pip install "veadk-python>=0.5.39" openai-codex +pip install "veadk-python[codex]" # openai-codex + the bundled Codex CLI binary python app.py # or: python -m app # open http://127.0.0.1:8000 ; POST /run_sse, or GET /ping -> {"status":"ok"} ``` @@ -99,8 +116,17 @@ tear the runtime down. - **Model**: the model in `MODEL_AGENT_*` is bridged to Codex; it does not need to be an OpenAI model — a Volcengine Ark chat model works. - **Tools / sandbox**: Codex runs tool calls (e.g. shell) in its own sandbox - inside the container. For tool-heavy agents that need filesystem/network - access, the runtime may need to be granted the corresponding permissions. + inside the container. The defaults are the safe ones — `workspace_write`, + `network_access=False`, and `approval_mode="deny_all"`, which refuses every + escalation. To let the agent reach the network, set + `CodexRuntimeConfig(sandbox="workspace_write", network_access=True)`; see the + [runtime docs](../../docs/content/docs/framework/agent/runtime.en.mdx). + Do **not** reach for `approval_mode="auto_review"` — it is not a review gate, + it auto-approves every escalation. +- **Runtime env vars override the Python config**: `VEADK_CODEX_SANDBOX`, + `VEADK_CODEX_APPROVAL_MODE`, `VEADK_CODEX_WORKSPACE_ROOT` and + `VEADK_CODEX_NETWORK_ACCESS` take precedence over `CodexRuntimeConfig`, so be + careful about what you pass to `--runtime_envs`. - **First request latency**: the Codex app-server binary is spawned on first use, so the first turn is slower than subsequent ones. - **Build time**: installing veadk + openai-codex from PyPI can take several diff --git a/examples/codex_runtime_on_agentkit/README.zh.md b/examples/codex_runtime_on_agentkit/README.zh.md index c8dc65641..7a4fe8480 100644 --- a/examples/codex_runtime_on_agentkit/README.zh.md +++ b/examples/codex_runtime_on_agentkit/README.zh.md @@ -6,6 +6,16 @@ > English version: [README.md](./README.md) +> **这个示例是什么：** 一份**部署参考**。它展示如何把一个 `runtime="codex"` 的 +> 智能体打包并发布到 AgentKit——依赖版本怎么钉、自带的 Codex 二进制怎么进镜像、 +> `agentkit config` 要传哪些参数。里面的智能体本身只是个占位，它做的一问一答 +> 并不是这个运行时擅长的场景：这类问题用 `runtime="adk"` 更快也更便宜。 +> 这个运行时真正的用处见 [`codex_data_analysis/`](../codex_data_analysis/) +> （模型写脚本、跑起来、读 traceback、自己改好）与 +> [`codex_ops_assistant/`](../codex_ops_assistant/)（同一个循环用在日志与指标上， +> 全程断网沙箱），或 +> [什么时候该用 codex 运行时](../../docs/content/docs/framework/agent/runtime.mdx#什么时候该用-codex-运行时)。 + ## 目录结构 ```text @@ -13,7 +23,7 @@ codex_runtime_on_agentkit/ ├── app.py # 部署入口（ADK Agent API 服务） ├── agents/ │ └── codex_agent/ # Agent —— Agent(runtime="codex") -├── requirements.txt # veadk-python>=0.5.39 + openai-codex +├── requirements.txt # veadk-python + openai-codex + fastapi/uvicorn ├── .env.example └── .dockerignore ``` @@ -28,6 +38,11 @@ codex_runtime_on_agentkit/ `openai-codex-cli-bin`——以 **manylinux wheel** 形式打包了 Codex 二进制，Linux 构建里无需单独装二进制。它当前是 pre-release，连同其二进制依赖都**钉死到精确的 预发布版本**，这样 `uv pip install` 无需全局 `--prerelease=allow` 也能装上。 + 这些 pin 与 veadk-python 的 `[codex]` extra 保持一致；这里不直接用该 extra， + 是因为 uv 只在**顶层**钉死精确预发布版本时才放行，通过 extra 传递则不行。 +- `fastapi` / `uvicorn` 也显式列出：`app.py` 直接 import `uvicorn`，runtime 的 + Responses→chat shim 两者都在模块级 import。目前它们能从 google-adk 传递解析到， + 但 adk 已经在把 web 依赖挪进 extra，所以 `[codex]` 和本文件都显式声明。 > codex 运行时自 **0.5.39** 起已包含在 `veadk-python`（PyPI）中，所以镜像通过默认的 > `uv pip install -r requirements.txt` 全部从 PyPI 安装——无需构建脚本或 git clone。 @@ -43,7 +58,7 @@ cp .env.example .env ## 2. 本地运行（可选） ```bash -pip install "veadk-python>=0.5.39" openai-codex +pip install "veadk-python[codex]" # openai-codex + 自带的 Codex CLI 二进制 python app.py # 或：python -m app # 打开 http://127.0.0.1:8000；POST /run_sse，或 GET /ping -> {"status":"ok"} ``` @@ -92,8 +107,14 @@ veadk agentkit invoke "你好，你叫什么" # 测试 - **模型**：`MODEL_AGENT_*` 的模型会被桥接给 Codex，不必是 OpenAI 模型——火山引擎 Ark 的 chat 模型即可。 -- **工具 / 沙箱**：Codex 在容器内用自己的沙箱执行工具（如 shell）。对需要文件系统/ - 网络访问的重工具 Agent，运行时可能需要授予相应权限。 +- **工具 / 沙箱**：Codex 在容器内用自己的沙箱执行工具（如 shell）。默认值是安全的那一档—— + `workspace_write` 沙箱、`network_access=False`、`approval_mode="deny_all"`（拒绝一切提权）。 + 需要访问网络时设置 `CodexRuntimeConfig(sandbox="workspace_write", network_access=True)`， + 详见[运行时文档](../../docs/content/docs/framework/agent/runtime.mdx)。 + **不要**改用 `approval_mode="auto_review"`——它不是人工复核，而是对每一次提权全自动批准。 +- **运行时环境变量会覆盖 Python 配置**：`VEADK_CODEX_SANDBOX`、`VEADK_CODEX_APPROVAL_MODE`、 + `VEADK_CODEX_WORKSPACE_ROOT`、`VEADK_CODEX_NETWORK_ACCESS` 的优先级高于 + `CodexRuntimeConfig`，因此要留意 `--runtime_envs` 里传了什么。 - **首请求延迟**：Codex app-server 二进制在首次使用时启动，首轮比后续慢。 - **构建耗时**：从 PyPI 安装 veadk + openai-codex 可能要几分钟；若 CLI 的构建等待超时， 重跑 `veadk agentkit launch` 会复用已缓存的镜像层，很快完成。 diff --git a/examples/codex_runtime_on_agentkit/requirements.txt b/examples/codex_runtime_on_agentkit/requirements.txt index c5eb1aff2..a995c1128 100644 --- a/examples/codex_runtime_on_agentkit/requirements.txt +++ b/examples/codex_runtime_on_agentkit/requirements.txt @@ -2,11 +2,19 @@ # # veadk-python >= 0.5.39 ships the codex runtime (veadk/runtime/codex). # -# openai-codex is NOT a veadk dependency, so it is listed explicitly. It is -# currently a pre-release, and so is its bundled-binary dependency -# openai-codex-cli-bin (the Codex CLI as a manylinux wheel). Both are pinned to -# exact pre-release versions so `uv pip install` accepts them without needing a -# global --prerelease=allow flag. +# This mirrors veadk-python's own `[codex]` extra. The extra is not used +# directly because uv refuses a *transitive* pre-release: openai-codex and its +# bundled-binary dependency openai-codex-cli-bin (the Codex CLI as a manylinux +# wheel) are pre-releases, and uv only accepts them when the exact pre-release +# version is pinned at the top level, as below. Keep these pins in sync with +# the `[codex]` extra in pyproject.toml. veadk-python>=0.5.39 openai-codex==0.1.0b3 openai-codex-cli-bin==0.137.0a4 + +# The Responses->chat shim (veadk/runtime/codex/proxy.py) imports these at +# module level, and app.py imports uvicorn directly. They resolve transitively +# through google-adk today, but adk has already moved other web deps behind +# extras on 2.x, so `[codex]` declares them and so does this file. +fastapi +uvicorn diff --git a/examples/codex_with_skill_and_mcp/README.md b/examples/codex_with_skill_and_mcp/README.md index 74e4644e0..7d146d81e 100644 --- a/examples/codex_with_skill_and_mcp/README.md +++ b/examples/codex_with_skill_and_mcp/README.md @@ -3,6 +3,18 @@ A `runtime="codex"` agent that uses **both a local skill and an MCP tool** on a chat backend (Volcengine Ark). +> **What this example is:** a **wiring reference**. It shows how a local skill +> and an MCP tool reach the model under `runtime="codex"`, using a deliberately +> trivial task (one weather lookup) so that the plumbing is the only thing on +> screen. Read it that way, not as a recommendation to use the codex runtime for +> that kind of task — a single tool call plus a formatted answer is exactly what +> `runtime="adk"` does faster and cheaper, without a Codex subprocess per turn. +> For what this runtime is actually *for* — a model that writes a script, runs +> it, reads the traceback and fixes it — see +> [`codex_data_analysis/`](../codex_data_analysis/) and +> [`codex_ops_assistant/`](../codex_ops_assistant/), or +> [when to use the codex runtime](../../docs/content/docs/framework/agent/runtime.en.mdx#when-to-use-the-codex-runtime). + ``` codex_with_skill_and_mcp/ ├── main.py # the agent + a sample run @@ -44,7 +56,7 @@ Both are handled by the runtime — the agent code is just normal tool wiring. ## Run ```bash -pip install openai-codex # bundles the Codex CLI binary +pip install "veadk-python[codex]" # openai-codex + the bundled Codex CLI binary # Ark (or another OpenAI-compatible chat) credentials: export MODEL_AGENT_API_KEY=... export MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 @@ -62,3 +74,13 @@ python examples/codex_with_skill_and_mcp/main.py tokens) and ADK interactive authentication requested during tool execution are supported. Authentication required before an MCP toolset can list tools still depends on the corresponding ADK/MCP client capability. +- `runtime="codex"` is a *sandboxed-execution* runtime, not a drop-in + replacement for the ADK flow. Part of the `Agent` surface is rejected outright + under it (`sub_agents`, `output_schema`, `planner`, `code_executor`, + `generate_content_config` beyond `system_instruction`, `include_contents="none"`, + `enable_supervisor`, and an explicit `model=`), and more is dropped with a + warning (`knowledgebase`, `example_store`, `skills_mode`, ...). See the + [support matrix](../../docs/content/docs/framework/agent/runtime.en.mdx#support-matrix). +- Note the distinction this example relies on: ADK's `SkillToolset` is bridged + into Codex's native skill system, but VeADK's own `Agent(skills_mode=...)` is + **not** — that one warns and has no effect. diff --git a/examples/codex_with_skill_and_mcp/README.zh.md b/examples/codex_with_skill_and_mcp/README.zh.md index 63292b70b..bd275185a 100644 --- a/examples/codex_with_skill_and_mcp/README.zh.md +++ b/examples/codex_with_skill_and_mcp/README.zh.md @@ -2,6 +2,16 @@ 一个 `runtime="codex"` 的 Agent，在 chat 后端（火山方舟）上**同时使用本地 skill 和 MCP 工具**。 +> **这个示例是什么：** 一份**接线参考**。它展示本地 skill 和 MCP 工具在 +> `runtime="codex"` 下分别怎么到达模型，任务故意选得极简（问一次天气）， +> 好让画面里只剩下管道本身。请照这个定位读它，而不要把它当成「这类任务应该用 +> codex 运行时」的推荐——一次固定的工具调用加一段格式化回答，正是 +> `runtime="adk"` 更快更便宜的场景，也不必每回合起一个 Codex 子进程。 +> 这个运行时**真正**的用处——模型写脚本、跑起来、读 traceback、自己改好—— +> 见 [`codex_data_analysis/`](../codex_data_analysis/) 与 +> [`codex_ops_assistant/`](../codex_ops_assistant/)，或 +> [什么时候该用 codex 运行时](../../docs/content/docs/framework/agent/runtime.mdx#什么时候该用-codex-运行时)。 + ``` codex_with_skill_and_mcp/ ├── main.py # Agent 定义 + 一次示例运行 @@ -36,7 +46,7 @@ Codex 接管了整轮（而不是 ADK 的 LLM flow），且只会说 Responses A ## 运行 ```bash -pip install openai-codex # 自带 Codex CLI 二进制 +pip install "veadk-python[codex]" # openai-codex + 自带的 Codex CLI 二进制 # 方舟（或其他 OpenAI 兼容 chat）凭证： export MODEL_AGENT_API_KEY=... export MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3 @@ -49,4 +59,5 @@ python examples/codex_with_skill_and_mcp/main.py - 工具由 runtime 的 shim 调度，但调用、结果、状态变更、确认和鉴权都会作为标准 ADK 事件进入 Session/Trace/UI。 - 支持静态鉴权（header / bearer token / ve-identity workload token）以及工具执行中触发的 ADK 交互式鉴权；MCP toolset 在列举工具前触发的鉴权仍取决于对应 ADK/MCP 客户端能力。 -``` +- `runtime="codex"` 是**沙箱执行运行时**，不是 ADK 执行流程的等价替代品。`Agent` 上有一部分配置在它下面会**直接报错**（`sub_agents`、`output_schema`、`planner`、`code_executor`、`system_instruction` 以外的 `generate_content_config`、`include_contents="none"`、`enable_supervisor`，以及显式传入的 `model=`），另一部分会被丢弃并告警（`knowledgebase`、`example_store`、`skills_mode` 等）。详见[支持矩阵](../../docs/content/docs/framework/agent/runtime.mdx#支持矩阵)。 +- 注意本例依赖的区别：ADK 的 `SkillToolset` 会被桥接进 Codex 原生 skill 系统，但 VeADK 自己的 `Agent(skills_mode=...)` **不会**——后者只会告警且不生效。 diff --git a/examples/codex_with_skill_and_mcp/main.py b/examples/codex_with_skill_and_mcp/main.py index d91a444e1..829efd873 100644 --- a/examples/codex_with_skill_and_mcp/main.py +++ b/examples/codex_with_skill_and_mcp/main.py @@ -29,7 +29,8 @@ python examples/codex_with_skill_and_mcp/main.py Requires: -- ``pip install openai-codex`` (bundles the Codex CLI binary). +- ``pip install "veadk-python[codex]"`` (openai-codex plus the bundled Codex + CLI binary). - Ark (or another OpenAI-compatible chat) credentials via ``MODEL_AGENT_API_KEY`` / ``MODEL_AGENT_API_BASE`` / ``MODEL_AGENT_NAME`` (see the repo .env.example). """ diff --git a/pyproject.toml b/pyproject.toml index 67a42303d..2f9fb4cd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,13 @@ dependencies = [ # them behind extras ([extensions]/[db]) on 2.x, so declare them directly to # stay compatible across google-adk >= 1.34.0 (including 2.x). "litellm>=1.83.7,<=1.83.14", # google-adk LiteLlm model ([extensions] on 2.x) - "sqlalchemy>=2,<3", # google-adk sessions ([db] on 2.x) + # The `asyncio` extra is what pulls greenlet, which SQLAlchemy's async + # engine requires and which ADK's DatabaseSessionService therefore needs. + # Plain `sqlalchemy` leaves it to a platform marker that does not cover + # macOS arm64, so a bare install there raised "the greenlet library is + # required" on the first session write to sqlite/mysql/postgres. CI never + # saw it: ubuntu-latest is x86_64, where the marker matches. + "sqlalchemy[asyncio]>=2,<3", # google-adk sessions ([db] on 2.x) "opentelemetry-exporter-otlp==1.37.0", "opentelemetry-instrumentation-logging>=0.56b0", "wrapt==1.17.2", # For patching built-in functions @@ -66,6 +72,12 @@ github-cicd = [ codex = [ "openai-codex==0.1.0b3", "openai-codex-cli-bin==0.137.0a4", + # The Responses->chat shim (veadk/runtime/codex/proxy.py) imports these at + # module level. They resolve transitively through google-adk today, but adk + # has already moved other web deps behind extras on 2.x, so declare them + # here rather than rely on that. + "fastapi", + "uvicorn", ] extensions = [ "redis>=5.0", # For Redis database diff --git a/pytest.ini b/pytest.ini index cd5d19229..1b351bbb6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,3 +4,10 @@ pythonpath = . addopts = -W ignore::pydantic.PydanticDeprecatedSince20 filterwarnings = ignore::UserWarning +# pytest-asyncio's default, pinned explicitly: every async test must carry +# `@pytest.mark.asyncio`. Leaving it unset makes the mode depend on the +# installed plugin version. +asyncio_mode = strict +markers = + piagent_smoke: real Pi binary/model smoke test (opt in with PIAGENT_RUN_SMOKE=1) + codex_smoke: real Codex binary/sandbox/socket smoke test, stubbed model backend (opt in with CODEX_RUN_SMOKE=1); binds real ports and spawns a subprocess, so it must not run under `pytest -n` diff --git a/tests/cli/test_cli_harness_open_source_defaults.py b/tests/cli/test_cli_harness_open_source_defaults.py index e5225cc4c..e28b46d42 100644 --- a/tests/cli/test_cli_harness_open_source_defaults.py +++ b/tests/cli/test_cli_harness_open_source_defaults.py @@ -43,7 +43,11 @@ def test_harness_dockerfile_uses_accelerated_source_with_official_fallback() -> in cli_harness._DOCKERFILE ) assert "https://github.com/volcengine/veadk-python.git" in cli_harness._DOCKERFILE - assert '"./src[harness]"' in cli_harness._DOCKERFILE + # The harness advertises `runtime: codex` in harness.yaml and honours a + # per-request runtime override, so the image must carry the codex extra or + # every such request fails with an ImportError on an already-deployed + # runtime. + assert '"./src[harness,codex]"' in cli_harness._DOCKERFILE old_package_path = "packages/" + "agentkit" + "-harness-python" assert old_package_path not in cli_harness._DOCKERFILE diff --git a/tests/runtime/codex/README.md b/tests/runtime/codex/README.md new file mode 100644 index 000000000..65ce54196 --- /dev/null +++ b/tests/runtime/codex/README.md @@ -0,0 +1,63 @@ +# Codex runtime tests + +## What runs where + +| File | Needs `openai-codex`? | Runs locally by default | +| --- | --- | --- | +| `test_codex_runtime.py` | no | yes | +| `test_codex_shim_rounds.py` | no | yes | +| `test_codex_tracing.py` | no (a stub SDK is installed) | yes | +| `../differential/` | no (a stub SDK is installed) | yes | +| `test_codex_runtime_sdk.py` | **yes** (`pytest.importorskip`) | **no — silently skipped** | +| `test_codex_sdk_protocol.py` | **yes** (`pytest.importorskip`) | **no — silently skipped** | + +The last two are the only tests that touch the real SDK types, and they are the +ones a developer machine is most likely to skip without noticing. `openai-codex` +is an optional extra; CI installs it (`uv sync --all-extras` in +`.github/workflows/unit-tests.yaml`), a checkout usually does not. A green local +run therefore does **not** mean the SDK contract holds. + +To run them locally: + +```bash +uv sync --all-extras # or: pip install 'openai-codex==0.1.0b3' +PYTHONPYCACHEPREFIX=/private/tmp/veadk-pycache \ + .venv/bin/python -m pytest tests/runtime/codex/test_codex_sdk_protocol.py -v +``` + +Confirm they are not skipping: + +```bash +.venv/bin/python -m pytest tests/runtime/codex -q -rs # -rs lists skip reasons +``` + +## Why the differential suite still runs without the SDK + +`veadk/runtime/codex/runtime.py` imports `openai_codex` at module scope, so +`Agent(runtime="codex")` is unimportable without the extra. The differential +harness installs a minimal stub into `sys.modules` +(`tests/runtime/differential/fake_codex_sdk.py::install_openai_codex_stub`) from +a *fixture*, never at import time — pytest finishes collection, and therefore +evaluates every `importorskip("openai_codex")`, before the first test runs, so +the stub cannot turn a legitimate skip into a spurious pass. + +The stub only replaces the names the runtime imports. `AsyncCodex` is always +replaced by `ShimDrivingCodex`, which POSTs a real `stream: True` +`/v1/responses` request at the real `ResponsesShim` over `httpx.ASGITransport` +(in-process, no socket, no Codex binary, xdist-safe) and reads its endpoint out +of the `config.toml` that `_prepare_codex_home` generated. + +## No network, no ports, no binary + +Nothing in this directory or in `../differential/` binds a port, spawns the +Codex CLI, or reaches the network — with one exception: +`test_codex_runtime.py::test_tool_executor_supports_stdio_mcp_toolset` spawns a +real Python subprocess from `examples/`. It is bounded by an explicit timeout so +it cannot hang a `pytest -n 16` run. + +`test_codex_shim_rounds.py` constructs `ResponsesShim` directly rather than +calling `get_shim`, so the process-global `_SHIMS` cache (and its uvicorn +servers) is never populated; an autouse fixture asserts that. The one test that +must exercise `get_shim` — the cache is what it tests — swaps `_SHIMS`/`_RETIRED` +for empty ones, restores them in a `finally` before that fixture runs, and stubs +`start()` so nothing binds a port. diff --git a/tests/runtime/codex/test_codex_runtime.py b/tests/runtime/codex/test_codex_runtime.py index bd721c4ea..2125693ac 100644 --- a/tests/runtime/codex/test_codex_runtime.py +++ b/tests/runtime/codex/test_codex_runtime.py @@ -21,7 +21,6 @@ from pathlib import Path from types import SimpleNamespace -import httpx import pytest from google.adk.agents.invocation_context import InvocationContext from google.adk.agents.llm_agent import LlmAgent @@ -39,18 +38,16 @@ from fastapi.openapi.models import APIKeyIn from veadk.runtime.base_runtime import resolve_system_append -from veadk.runtime.agent_transfer import build_transfer_tool from veadk.runtime.codex.config import CodexRuntimeConfig from veadk.runtime.codex.config import codex_subprocess_env from veadk.runtime.codex.config import toml_string -from veadk.runtime.codex.proxy import ResponsesShim -from veadk.runtime.codex.tools_bridge import add_tool_to_bundle from veadk.runtime.codex.tools_bridge import build_executable_tools from veadk.runtime.codex.tools_bridge import close_toolsets from veadk.runtime.codex.tools_bridge import resume_authenticated_tools from veadk.runtime.codex.tools_bridge import resume_confirmed_tools from veadk.runtime.codex.translate import build_prompt from veadk.runtime.codex.translate import build_input_attachments +from veadk.runtime.codex.translate import build_turn_usage_metadata from veadk.runtime.codex.translate import notification_to_events @@ -331,7 +328,12 @@ def test_native_plan_error_and_turn_complete_are_observable() -> None: (), { "model_dump": lambda self: { - "error": {"code": "backend", "message": "retrying"}, + # A real TurnError has no `code` field; the classification comes + # from `codex_error_info`. + "error": { + "message": "retrying", + "codex_error_info": "contextWindowExceeded", + }, "will_retry": True, } }, @@ -351,31 +353,172 @@ def test_native_plan_error_and_turn_complete_are_observable() -> None: complete_event = notification_to_events(completed, "agent", "inv")[0] assert plan_event.custom_metadata["plan"][0]["step"] == "test" - assert error_event.error_code == "backend" + assert error_event.error_code == "contextWindowExceeded" assert error_event.custom_metadata["will_retry"] is True assert complete_event.turn_complete is True assert complete_event.custom_metadata["turn_id"] == "turn-1" + # Lifecycle markers are `partial=True` for a clean turn, so they are not + # final responses and never reach `output_key` or the persisted session. + assert plan_event.partial is True + assert plan_event.is_final_response() is False + assert complete_event.partial is True + + +# The exact payload `thread/tokenUsage/updated` carries: `last` (the model call +# that just finished) and `total` (cumulative for the thread), each a +# TokenUsageBreakdown, plus a sibling `model_context_window`. +_TOKEN_USAGE_PAYLOAD = { + "turn_id": "turn-1", + "model_context_window": 128000, + "token_usage": { + "last": { + "input_tokens": 10, + "cached_input_tokens": 2, + "output_tokens": 4, + "reasoning_output_tokens": 1, + "total_tokens": 14, + }, + "total": { + "input_tokens": 30, + "cached_input_tokens": 6, + "output_tokens": 9, + "reasoning_output_tokens": 3, + "total_tokens": 39, + }, + }, +} -def test_native_token_usage_is_observable() -> None: - usage = type( +def _token_usage_notification() -> object: + return type( "ThreadTokenUsageUpdatedNotification", (), - { - "model_dump": lambda self: { - "turn_id": "turn-1", - "token_usage": { - "last": {"input_tokens": 10, "output_tokens": 4}, - "total": {"input_tokens": 10, "output_tokens": 4}, - }, - } - }, + {"model_dump": lambda self: dict(_TOKEN_USAGE_PAYLOAD)}, )() - event = notification_to_events(usage, "agent", "inv")[0] +def test_token_usage_notification_populates_usage_metadata() -> None: + """Token accounting must reach `usage_metadata`, not just a log line. + + The previous version of this test asserted only + `custom_metadata["token_usage"]["total"]["output_tokens"]`, a key whose only + repo-wide reader is a `logger.info`. That locked the bug in as the spec: + `usage_metadata` is the field every real consumer reads (portal metrics, + the trace exporter, the frontend token counter), and it was empty. + + The lifecycle event itself deliberately carries no `usage_metadata` -- it + fires once per model call and is `partial`, so it is never persisted, and + every consumer sums `usage_metadata` across events with no dedupe. The + cumulative figure is attached once, to the merged final response. + """ + event = notification_to_events(_token_usage_notification(), "agent", "inv")[0] + + # Still a real UI contract: the raw mapping stays readable, and it is the + # only place `reasoning_output_tokens` survives. assert event.custom_metadata["codex_event_type"] == "token_usage" - assert event.custom_metadata["token_usage"]["total"]["output_tokens"] == 4 + assert event.custom_metadata["token_usage"] == _TOKEN_USAGE_PAYLOAD["token_usage"] + assert event.usage_metadata is None + + usage = build_turn_usage_metadata(event.custom_metadata["token_usage"]) + assert usage is not None + assert usage.prompt_token_count == 30 + assert usage.candidates_token_count == 9 + assert usage.total_token_count == 39 + assert usage.cached_content_token_count == 6 + # Codex nests reasoning inside output; genai treats thoughts as disjoint + # from candidates, so mapping it would double-count any recomputed total. + assert usage.thoughts_token_count is None + + +def test_turn_usage_metadata_falls_back_to_last_when_total_is_absent() -> None: + usage = build_turn_usage_metadata( + {"last": dict(_TOKEN_USAGE_PAYLOAD["token_usage"]["last"])} + ) + + assert usage is not None + assert (usage.prompt_token_count, usage.candidates_token_count) == (10, 4) + + +def test_turn_usage_metadata_degrades_to_none_rather_than_zeroes() -> None: + """A malformed payload must not pollute token histograms with zeroes.""" + assert build_turn_usage_metadata(None) is None + assert build_turn_usage_metadata({}) is None + assert build_turn_usage_metadata({"total": {"unrelated": 1}}) is None + + +@pytest.fixture +def fresh_global_meter_provider(monkeypatch): + """Give this test an isolated OpenTelemetry global meter provider. + + ``PortalMetricRecorder`` builds its instruments in ``__init__`` from + whatever provider is installed *then*, and measurements taken before a real + provider is installed are dropped, so the provider has to be in place first + and torn down afterwards. + """ + from opentelemetry.metrics import _internal as metrics_internal + from opentelemetry.sdk import metrics as metrics_sdk + from opentelemetry.util._once import Once + + proxy_provider = metrics_internal._PROXY_METER_PROVIDER + monkeypatch.setattr(metrics_internal, "_METER_PROVIDER", None) + monkeypatch.setattr(metrics_internal, "_METER_PROVIDER_SET_ONCE", Once()) + monkeypatch.setattr(proxy_provider, "_real_meter_provider", None) + monkeypatch.setattr(proxy_provider, "_meters", []) + yield + provider = metrics_internal._METER_PROVIDER + if isinstance(provider, metrics_sdk.MeterProvider): + provider.shutdown() + + +def test_token_usage_reaches_portal_metrics(fresh_global_meter_provider) -> None: + """The real downstream: `usage_metadata` -> `record_call_llm` -> tokens. + + `portal_metrics.record_call_llm` is gated entirely on + `llm_response.usage_metadata`; with it unset the recorder emits no token + histogram samples and not even the invocation counter. This walks the whole + chain that the old assertion skipped. + """ + from opentelemetry import metrics as metrics_api + from opentelemetry.sdk import metrics as metrics_sdk + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + from veadk.runtime.model_callbacks import event_to_llm_response + from veadk.tracing.telemetry.portal_metrics import PortalMetricRecorder + + event = notification_to_events(_token_usage_notification(), "agent", "inv")[0] + event.usage_metadata = build_turn_usage_metadata( + event.custom_metadata["token_usage"] + ) + llm_response = event_to_llm_response(event) + assert llm_response.usage_metadata is not None + + reader = InMemoryMetricReader() + provider = metrics_sdk.MeterProvider(metric_readers=[reader]) + metrics_api.set_meter_provider(provider) + recorder = PortalMetricRecorder(name="codex-token-usage-test") + + ctx = SimpleNamespace( + run_config=None, + agent=SimpleNamespace(model_api_base="https://backend.invalid/v1"), + ) + recorder.record_call_llm( + ctx, "event-id", SimpleNamespace(model="scripted-model"), llm_response + ) + provider.force_flush() + + recorded: dict[str, int] = {} + for resource in reader.get_metrics_data().resource_metrics: + for scope in resource.scope_metrics: + for metric in scope.metrics: + for point in metric.data.data_points: + token_type = dict(point.attributes).get("gen_ai_token_type") + if metric.name == "gen_ai.client.token.usage" and token_type: + recorded[token_type] = getattr(point, "sum", None) or 0 + + assert recorded.get("input"), f"no input tokens recorded: {recorded}" + assert recorded.get("output"), f"no output tokens recorded: {recorded}" + assert recorded["input"] == 30 + assert recorded["output"] == 9 @pytest.mark.asyncio @@ -718,320 +861,18 @@ async def test_tool_executor_supports_stdio_mcp_toolset() -> None: ) ctx = _ctx(agent) - bundle = await build_executable_tools(agent, ctx) + # This is the one test in the tree that spawns a real subprocess. Without a + # deadline a stuck MCP server hangs a `pytest -n 16` worker indefinitely. + bundle = await asyncio.wait_for(build_executable_tools(agent, ctx), timeout=60) try: output = json.loads( - await bundle.executors["get_order_status"]( - {"order_id": "A10086"}, "call-mcp" + await asyncio.wait_for( + bundle.executors["get_order_status"]( + {"order_id": "A10086"}, "call-mcp" + ), + timeout=60, ) ) assert output["structuredContent"]["status"] == "paid" finally: - await close_toolsets(bundle.opened_toolsets) - - -@pytest.mark.asyncio -async def test_transfer_tool_executor_emits_adk_transfer_action() -> None: - worker = LlmAgent(name="worker", model="gemini-2.5-flash") - agent = LlmAgent( - name="agent", - model="gemini-2.5-flash", - sub_agents=[worker], - ) - ctx = _ctx(agent) - emitted: list[Event] = [] - - async def emit(event: Event) -> None: - emitted.append(event) - - bundle = await build_executable_tools(agent, ctx, event_sink=emit) - try: - add_tool_to_bundle( - bundle, - build_transfer_tool([worker]), - ctx, - event_sink=emit, - ) - output = json.loads( - await bundle.executors["transfer_to_agent"]( - {"agent_name": "worker"}, "call-transfer" - ) - ) - finally: - await close_toolsets(bundle.opened_toolsets) - - assert output["status"] == "transferred" - assert output["agent_name"] == "worker" - assert bundle.specs[0]["parameters"]["properties"]["agent_name"]["enum"] == [ - "worker" - ] - assert any(event.actions.transfer_to_agent == "worker" for event in emitted) - - -@pytest.mark.asyncio -async def test_shim_routes_concurrent_turns_to_their_own_executors( - monkeypatch, -) -> None: - shim = ResponsesShim("https://backend.invalid/v1", "backend-key") - calls: list[tuple[str, str]] = [] - - async def executor_a(args, call_id): - await asyncio.sleep(0.01) - calls.append(("a", call_id)) - return json.dumps({"owner": "a"}) - - async def executor_b(args, call_id): - calls.append(("b", call_id)) - return json.dumps({"owner": "b"}) - - token_a = shim.register_turn( - [{"type": "function", "name": "tool_a", "parameters": {}}], - {"tool_a": executor_a}, - ) - token_b = shim.register_turn( - [{"type": "function", "name": "tool_b", "parameters": {}}], - {"tool_b": executor_b}, - ) - - async def fake_aresponses(**kwargs): - conversation = kwargs["input"] - if any(item.get("type") == "function_call_output" for item in conversation): - return { - "id": "resp-final", - "model": "model", - "output": [ - { - "id": "msg", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "done"}], - } - ], - } - tool = next( - item for item in kwargs["tools"] if item["name"] in {"tool_a", "tool_b"} - ) - suffix = tool["name"][-1] - return { - "id": f"resp-{suffix}", - "model": "model", - "output": [ - { - "id": f"fc-{suffix}", - "call_id": f"call-{suffix}", - "type": "function_call", - "name": tool["name"], - "arguments": "{}", - "status": "completed", - } - ], - } - - monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", fake_aresponses) - transport = httpx.ASGITransport(app=shim._app) - async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: - body = { - "model": "model", - "stream": False, - "input": [{"type": "message", "role": "user", "content": "go"}], - } - response_a, response_b = await asyncio.gather( - client.post( - "/v1/responses", - headers={"Authorization": f"Bearer {token_a}"}, - json=body, - ), - client.post( - "/v1/responses", - headers={"Authorization": f"Bearer {token_b}"}, - json=body, - ), - ) - - assert response_a.status_code == 200 - assert response_b.status_code == 200 - assert sorted(calls) == [("a", "call-a"), ("b", "call-b")] - - -@pytest.mark.asyncio -async def test_shim_rejects_unknown_invocation_token() -> None: - shim = ResponsesShim("https://backend.invalid/v1", "backend-key") - transport = httpx.ASGITransport(app=shim._app) - async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: - response = await client.post( - "/v1/responses", - headers={"Authorization": "Bearer unknown"}, - json={"model": "model", "input": []}, - ) - assert response.status_code == 401 - - -@pytest.mark.asyncio -async def test_shim_completes_turn_after_transfer_without_second_model_call( - monkeypatch, -) -> None: - shim = ResponsesShim("https://backend.invalid/v1", "backend-key") - backend_calls = 0 - - async def executor(args, call_id): - return json.dumps( - { - "status": "transferred", - "call_id": call_id, - "agent_name": args["agent_name"], - } - ) - - token = shim.register_turn( - [{"type": "function", "name": "transfer_to_agent", "parameters": {}}], - {"transfer_to_agent": executor}, - ) - - async def fake_aresponses(**kwargs): - nonlocal backend_calls - backend_calls += 1 - assert not any( - item.get("type") == "function_call_output" for item in kwargs["input"] - ) - return { - "id": "transfer-response", - "model": "model", - "output": [ - { - "id": "fc-transfer", - "call_id": "call-transfer", - "type": "function_call", - "name": "transfer_to_agent", - "arguments": json.dumps({"agent_name": "worker"}), - "status": "completed", - } - ], - } - - monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", fake_aresponses) - transport = httpx.ASGITransport(app=shim._app) - async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: - response = await client.post( - "/v1/responses", - headers={"Authorization": f"Bearer {token}"}, - json={ - "model": "model", - "input": [{"type": "message", "role": "user", "content": "go"}], - }, - ) - - assert response.status_code == 200 - assert backend_calls == 1 - body = response.json() - assert body["status"] == "completed" - assert body["output"][0]["type"] == "message" - - -@pytest.mark.asyncio -async def test_shim_reports_tool_iteration_budget_instead_of_dropping_call( - monkeypatch, -) -> None: - shim = ResponsesShim("https://backend.invalid/v1", "backend-key") - - async def executor(args, call_id): - return "{}" - - token = shim.register_turn( - [{"type": "function", "name": "loop", "parameters": {}}], - {"loop": executor}, - max_tool_iterations=1, - ) - - async def always_calls_tool(**kwargs): - return { - "id": "resp", - "model": "model", - "output": [ - { - "id": "fc", - "call_id": "call-loop", - "type": "function_call", - "name": "loop", - "arguments": "{}", - "status": "completed", - } - ], - } - - monkeypatch.setattr( - "veadk.runtime.codex.proxy.litellm.aresponses", always_calls_tool - ) - transport = httpx.ASGITransport(app=shim._app) - async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: - response = await client.post( - "/v1/responses", - headers={"Authorization": f"Bearer {token}"}, - json={ - "model": "model", - "input": [{"type": "message", "role": "user", "content": "go"}], - }, - ) - - assert response.status_code == 409 - assert response.json()["error"]["type"] == "tool_iteration_limit" - - -@pytest.mark.asyncio -async def test_shim_rejects_invalid_tool_json_without_calling_executor( - monkeypatch, -) -> None: - shim = ResponsesShim("https://backend.invalid/v1", "backend-key") - called = False - - async def executor(args, call_id): - nonlocal called - called = True - return "{}" - - token = shim.register_turn( - [{"type": "function", "name": "parse", "parameters": {}}], - {"parse": executor}, - ) - - async def fake_aresponses(**kwargs): - if any(item.get("type") == "function_call_output" for item in kwargs["input"]): - return { - "id": "final", - "model": "model", - "output": [ - { - "id": "msg", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": "handled"}], - } - ], - } - return { - "id": "tool", - "model": "model", - "output": [ - { - "id": "fc", - "call_id": "call-invalid", - "type": "function_call", - "name": "parse", - "arguments": "{not-json", - "status": "completed", - } - ], - } - - monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", fake_aresponses) - transport = httpx.ASGITransport(app=shim._app) - async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: - response = await client.post( - "/v1/responses", - headers={"Authorization": f"Bearer {token}"}, - json={"model": "model", "input": []}, - ) - - assert response.status_code == 200 - assert called is False + await asyncio.wait_for(close_toolsets(bundle.opened_toolsets), timeout=30) diff --git a/tests/runtime/codex/test_codex_runtime_sdk.py b/tests/runtime/codex/test_codex_runtime_sdk.py index 83a3f057f..ffe9730ea 100644 --- a/tests/runtime/codex/test_codex_runtime_sdk.py +++ b/tests/runtime/codex/test_codex_runtime_sdk.py @@ -29,6 +29,7 @@ from veadk.runtime.codex.config import CodexRuntimeConfig # noqa: E402 from veadk.runtime.codex.runtime import CodexRuntime # noqa: E402 +from veadk.runtime.codex.runtime import _TOOL_AVAILABILITY_NOTE # noqa: E402 from veadk.runtime.codex.runtime import _prepare_workspace # noqa: E402 @@ -44,15 +45,22 @@ def register_turn( specs, executors, *, - max_tool_iterations, - invocation_id, + max_tool_iterations=None, + invocation_id="", + model_extra_config=None, + on_model_call=None, ): + # Keyword-only params must carry defaults: the production signature has + # grown twice, and a required keyword here turns a runtime change into + # an opaque TypeError in an unrelated test. self.registered.append( { "specs": specs, "executors": executors, "max_tool_iterations": max_tool_iterations, "invocation_id": invocation_id, + "model_extra_config": model_extra_config, + "on_model_call": on_model_call, } ) return "opaque-turn-token" @@ -60,6 +68,16 @@ def register_turn( def unregister_turn(self, token): self.unregistered.append(token) + def turn_marker(self, token): + # Mirrors the real shim: an opaque per-turn marker the runtime embeds + # in the Codex prompt, and "" for a token the shim does not know. + return "turn-marker" if token == "opaque-turn-token" else "" + + def turn_error(self, token): + # The real shim reports an exception raised inside it that aborted the + # turn; nothing fails inside this fake, so there is never one. + return None + class _EmptyStream: def __aiter__(self): @@ -122,6 +140,12 @@ class _Agent: model_name = "test-model" model_api_base = "https://backend.invalid/v1" model_api_key = "backend-secret" + # `Agent.model_extra_config` is a real field (default_factory=dict), so it + # is always present on an agent the runtime is handed. The runtime forwards + # it to `shim.register_turn`; without it here the fake diverges from the + # production contract and the runtime raises AttributeError before reaching + # anything this test is about. + model_extra_config = {} codex_runtime_config = CodexRuntimeConfig() @@ -129,6 +153,11 @@ class _Context(SimpleNamespace): def _get_events(self, **kwargs): return list(self.session.events) + def increment_llm_call_count(self): + # Without this attribute the runtime's `max_llm_calls` charging + # silently no-ops and the SDK contract test covers nothing. + self.llm_call_count = getattr(self, "llm_call_count", 0) + 1 + @pytest.mark.asyncio async def test_runtime_passes_isolated_config_and_safe_sdk_controls( @@ -177,7 +206,16 @@ async def fake_get_shim(api_base, api_key): finally: runtime_logger.removeHandler(caplog.handler) - assert events == [] + # An empty Codex stream still yields exactly one event: the merged + # per-turn response. After-model callbacks have to run on every turn, and + # when nothing durable was emitted there is no tool event to fold the + # `state_delta`/`usage_metadata` bookkeeping onto and nothing for a + # contentless event to clobber -- so the runtime emits it rather than + # dropping it (the final `else` in `CodexRuntime.run_async`'s merge). + assert len(events) == 1, events + assert events[0].content is None + assert events[0].author == "agent" + assert events[0].invocation_id == "inv-sdk" assert shim.registered[0]["invocation_id"] == "inv-sdk" assert shim.unregistered == ["opaque-turn-token"] sdk_config = _FakeAsyncCodex.calls["config"] @@ -187,10 +225,28 @@ async def fake_get_shim(api_base, api_key): _FakeAsyncCodex.calls["thread_start"]["approval_mode"] is ApprovalMode.deny_all ) assert _FakeAsyncCodex.calls["thread_start"]["sandbox"] is Sandbox.workspace_write - assert _FakeAsyncCodex.calls["thread_start"]["base_instructions"] + # `base_instructions` *replaces* Codex's 20.9KB built-in system prompt, so + # the runtime deliberately never sends it; the agent identity rides along + # with the instruction on the developer channel instead. + assert "base_instructions" not in _FakeAsyncCodex.calls["thread_start"] + # Identity, then the agent's instruction, then the runtime's note about the + # two tools Codex's own prompt gets wrong here (its content is asserted in + # `test_codex_turn_contract.py`; this row is about ordering and joining). assert _FakeAsyncCodex.calls["thread_start"]["developer_instructions"] == ( - "Follow the contract." + "Your name is agent.\n\nSDK contract agent\n\nFollow the contract.\n\n" + + _TOOL_AVAILABILITY_NOTE ) + # `on_model_call` is what makes `RunConfig.max_llm_calls` fire at all for + # runtime="codex": ADK enforces the budget only through + # `increment_llm_call_count`, which its own LLM flow -- the one this + # runtime replaces -- would normally call. The real shim invokes this hook + # once per backend model call; `_FakeShim` never reaches a backend, so + # assert the runtime *wired* it and that invoking it charges the context. + on_model_call = shim.registered[0]["on_model_call"] + assert on_model_call is not None, "max_llm_calls can never fire for this runtime" + assert not hasattr(ctx, "llm_call_count") + on_model_call() + assert ctx.llm_call_count == 1, "the invocation was never charged an LLM call" messages = "\n".join(record.getMessage() for record in caplog.records) assert "codex_runtime_start invocation_id=inv-sdk" in messages assert "codex_runtime_complete invocation_id=inv-sdk status=completed" in messages @@ -257,18 +313,21 @@ def context(session_id): ), ) - first, first_cleanup = _prepare_workspace(config, context("session-a")) - repeated, repeated_cleanup = _prepare_workspace(config, context("session-a")) - second, second_cleanup = _prepare_workspace(config, context("session-b")) + first = _prepare_workspace(config, context("session-a")) + repeated = _prepare_workspace(config, context("session-a")) + second = _prepare_workspace(config, context("session-b")) assert first == repeated assert first != second - assert first_cleanup is repeated_cleanup is second_cleanup is False + + # `_prepare_workspace` returns a plain path: the old second tuple element + # was always False, which made four rmtree call sites dead code. Session + # workspaces outlive a turn and are reaped on an idle TTL instead. + assert isinstance(first, str) shared_config = CodexRuntimeConfig( workspace_root=str(tmp_path / "shared"), reuse_workspace=True, ) - shared, cleanup = _prepare_workspace(shared_config, context("session-c")) + shared = _prepare_workspace(shared_config, context("session-c")) assert shared == str(tmp_path / "shared") - assert cleanup is False diff --git a/tests/runtime/codex/test_codex_runtime_smoke.py b/tests/runtime/codex/test_codex_runtime_smoke.py new file mode 100644 index 000000000..535a08d1b --- /dev/null +++ b/tests/runtime/codex/test_codex_runtime_smoke.py @@ -0,0 +1,547 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One real end-to-end Codex turn: real binary, real sandbox, real shim socket. + +Every other Codex test in this tree stubs at least one of three boundaries: the +model backend (``litellm.aresponses`` monkeypatched), the Codex subprocess +(faked), and the socket (``httpx.ASGITransport``, so ``ResponsesShim.start()`` +is never called anywhere else). That leaves a class of deployment-blocking +failures with zero coverage: the Codex CLI binary spawning at all, OS sandbox +availability (macOS seatbelt / Linux landlock+seccomp), real port binding, the +shim's start/stop lifecycle, and whether the ``config.toml`` written by +``_prepare_codex_home`` is actually accepted by the pinned binary. + +This test closes exactly that gap and nothing else. **There is no real model**: +the shim is pointed at a stub Responses backend served on another loopback +port, which replies with a canned script. So the only real things here are the +blockers above -- no network, no credentials, no model nondeterminism. + +What the canned script drives, in order: + +1. a call to *Codex's own* ``shell`` tool (``/bin/echo``), which Codex executes + inside the OS sandbox -- this is the sandbox-establishment probe, and its + output is read back off the next request; +2. a call to the agent's ADK tool, which the *shim* executes (Codex never sees + it), proving the ADK tool path works over a real socket; +3. a final assistant message. + +Opt in with ``CODEX_RUN_SMOKE=1``; the ``codex_smoke`` marker (registered in +``pytest.ini``) keeps it out of the parallel ``pytest -n 16`` CI job, which it +must never join: it binds two real ports and spawns a real subprocess. + + CODEX_RUN_SMOKE=1 pytest tests/runtime/codex/test_codex_runtime_smoke.py \ + -p no:xdist -s + +Note: ``pytest_configure`` is deliberately NOT defined here. pytest only +collects that hook from ``conftest.py`` and plugins, never from a test module, +so a copy of the version in some other test files would be dead code. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import importlib.util +import json +import os +import platform +import shutil +import tempfile +import time +import uuid +from pathlib import Path +from typing import Any + +import pytest +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +#: Echoed by Codex's own sandboxed command tool. Its presence in the tool +#: output is the proof that the OS sandbox was established and ran a command. +_SANDBOX_PROBE_MARKER = "veadk-codex-sandbox-probe-ok" +#: Written by the same probe: `sandbox="read_only"` must refuse the write, so +#: which of these comes back says whether the sandbox is actually enforcing. +_WRITE_ALLOWED_MARKER = "veadk-codex-write-allowed" +_WRITE_DENIED_MARKER = "veadk-codex-write-denied" +#: Path the probe tries to create, relative to the turn's workspace (cwd). +_WRITE_PROBE_FILE = "veadk-codex-write-probe" +#: Passed to the ADK tool by the stub backend and recorded by the tool body. +_TOOL_MARKER = "veadk-codex-smoke-tool-marker" +#: The canned final assistant message. +_FINAL_ANSWER = "veadk-codex-smoke-final-answer" +#: Name of the agent's ADK tool. Must match the function's `__name__`. +_ADK_TOOL_NAME = "veadk_smoke_record_marker" +#: Hard ceiling on the whole run, so a wedged subprocess fails the test instead +#: of hanging CI. +_RUN_TIMEOUT_SECONDS = 60.0 + + +def _codex_binary() -> Path | None: + """Resolve the Codex CLI the SDK would spawn, or None if unavailable.""" + try: + from codex_cli_bin import bundled_codex_path # type: ignore[import-not-found] + except Exception: # noqa: BLE001 - the bin package is an optional extra + found = shutil.which("codex") + return Path(found) if found else None + try: + path = Path(bundled_codex_path()) + except Exception: # noqa: BLE001 + return None + return path if path.exists() and os.access(path, os.X_OK) else None + + +def _skip_reason() -> str | None: + """Three distinct skips: platform, SDK, binary. Never one blurred message.""" + system = platform.system() + if system not in ("Darwin", "Linux"): + return ( + f"codex smoke: unsupported platform {system!r}; the runtime's OS " + "sandbox is macOS seatbelt / Linux landlock+seccomp only" + ) + if importlib.util.find_spec("openai_codex") is None: + return ( + "codex smoke: the `openai-codex` SDK is not installed " + "(install the `codex` extra: uv sync --all-extras)" + ) + if _codex_binary() is None: + return ( + "codex smoke: no runnable Codex CLI binary " + "(`openai-codex-cli-bin` missing, or its bundled binary is not " + "executable on this platform)" + ) + return None + + +class _StubResponsesBackend: + """A scripted Responses API on 127.0.0.1:0, standing in for the model. + + The shim forwards through ``litellm.aresponses`` with + ``custom_llm_provider="openai"``, which POSTs to ``{api_base}/responses`` + -- so this serves ``/v1/responses`` and speaks the Responses wire format, + not chat completions. + + It is a state machine over the requests of a single turn, and it adapts to + what Codex actually advertises: the sandbox probe is only sent if Codex + offers a ``shell``-shaped function tool, and the ADK tool call is only sent + once the shim has advertised the tool. + """ + + def __init__(self) -> None: + self.url: str | None = None + self.requests: list[dict[str, Any]] = [] + #: Raw output Codex reported for the sandboxed `shell` call, once seen. + self.sandbox_probe_output: str | None = None + self.shell_tool_name: str | None = None + self._shell_call_id: str | None = None + self._adk_call_sent = False + self._server: uvicorn.Server | None = None + self._task: asyncio.Task[Any] | None = None + + app = FastAPI() + + @app.post("/v1/responses") + async def responses(request: Request) -> Any: # noqa: D401 + body = await request.json() + self.requests.append(body) + self._capture_sandbox_probe(body) + return JSONResponse(self._script(body)) + + self._app = app + + async def start(self) -> str: + config = uvicorn.Config( + self._app, host="127.0.0.1", port=0, log_level="warning", lifespan="off" + ) + server = uvicorn.Server(config) + server.install_signal_handlers = lambda: None # type: ignore[method-assign] + self._server = server + self._task = asyncio.create_task(server.serve()) + deadline = time.monotonic() + 10 + while not server.started: + if self._task.done(): + raise RuntimeError("stub backend exited before binding") + if time.monotonic() >= deadline: + raise TimeoutError("stub backend did not bind within 10s") + await asyncio.sleep(0.02) + port = server.servers[0].sockets[0].getsockname()[1] + self.url = f"http://127.0.0.1:{port}" + return self.url + + async def stop(self) -> None: + if self._server is not None: + self._server.should_exit = True + if self._task is not None: + with contextlib.suppress(Exception): + await asyncio.wait_for(self._task, 10) + + # -- scripting --------------------------------------------------------- + + def _script(self, body: dict[str, Any]) -> dict[str, Any]: + names = { + tool.get("name") + for tool in (body.get("tools") or []) + if isinstance(tool, dict) + } + if self.shell_tool_name is None and self._shell_call_id is None: + probe = _sandbox_probe_call(body.get("tools")) + if probe is not None: + self.shell_tool_name, arguments = probe + self._shell_call_id = f"call_{uuid.uuid4().hex[:16]}" + return self._response( + body, + [ + self._function_call( + self.shell_tool_name, + arguments, + call_id=self._shell_call_id, + ) + ], + ) + if not self._adk_call_sent and _ADK_TOOL_NAME in names: + self._adk_call_sent = True + return self._response( + body, + [self._function_call(_ADK_TOOL_NAME, {"marker": _TOOL_MARKER})], + ) + return self._response(body, [self._message(_FINAL_ANSWER)]) + + def _capture_sandbox_probe(self, body: dict[str, Any]) -> None: + """Read the sandboxed `shell` result out of the replayed conversation.""" + if self._shell_call_id is None or self.sandbox_probe_output is not None: + return + for item in body.get("input") or []: + if not isinstance(item, dict): + continue + if item.get("type") != "function_call_output": + continue + if item.get("call_id") != self._shell_call_id: + continue + output = item.get("output") + self.sandbox_probe_output = ( + output if isinstance(output, str) else json.dumps(output) + ) + return + + @staticmethod + def _function_call( + name: str, arguments: dict[str, Any], *, call_id: str | None = None + ) -> dict[str, Any]: + cid = call_id or f"call_{uuid.uuid4().hex[:16]}" + return { + "type": "function_call", + "id": f"fc_{uuid.uuid4().hex[:16]}", + "call_id": cid, + "name": name, + "arguments": json.dumps(arguments), + "status": "completed", + } + + @staticmethod + def _message(text: str) -> dict[str, Any]: + return { + "type": "message", + "id": f"msg_{uuid.uuid4().hex[:16]}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + + @staticmethod + def _response(body: dict[str, Any], output: list[dict[str, Any]]) -> dict[str, Any]: + return { + "id": f"resp_{uuid.uuid4().hex[:16]}", + "object": "response", + "created_at": int(time.time()), + "model": body.get("model", "stub-model"), + "status": "completed", + "output": output, + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 11, + "output_tokens": 7, + "total_tokens": 18, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +#: One command, two answers: it echoes a marker (proving the sandbox was +#: established and executed something) and then tries a write (proving +#: `sandbox="read_only"` is actually enforced, not merely configured). +_PROBE_SCRIPT = ( + f"/bin/echo {_SANDBOX_PROBE_MARKER}; " + f"if /usr/bin/touch ./{_WRITE_PROBE_FILE} 2>/dev/null; " + f"then /bin/echo {_WRITE_ALLOWED_MARKER}; " + f"else /bin/echo {_WRITE_DENIED_MARKER}; fi" +) + + +def _sandbox_probe_call(tools: Any) -> tuple[str, dict[str, Any]] | None: + """Codex's own command-execution tool and arguments for the probe. + + Matched structurally rather than by name: the pinned CLI (0.137) offers + unified exec as ``exec_command`` with a ``cmd`` *string*, while older and + other model families offer ``shell`` with a ``command`` *array*. Both + shapes are handled so a CLI bump does not silently stop exercising the + sandbox -- if neither is found the test fails loudly instead. + + The shim drops every non-``function`` tool before forwarding, so only + function-shaped variants can be probed at all. + """ + for tool in tools or []: + if not isinstance(tool, dict) or tool.get("type") != "function": + continue + name = tool.get("name") + if not isinstance(name, str): + continue + properties = (tool.get("parameters") or {}).get("properties") or {} + if properties.get("cmd", {}).get("type") == "string": + arguments: dict[str, Any] = {"cmd": _PROBE_SCRIPT, "login": False} + if "yield_time_ms" in properties: + arguments["yield_time_ms"] = 5000 + return name, arguments + if properties.get("command", {}).get("type") == "array": + return name, {"command": ["/bin/sh", "-c", _PROBE_SCRIPT]} + return None + + +@pytest.mark.codex_smoke +@pytest.mark.asyncio +async def test_real_codex_binary_completes_one_tool_using_turn(monkeypatch) -> None: + """Drive a real Codex subprocess through Runner over a real shim socket.""" + if os.getenv("CODEX_RUN_SMOKE") != "1": + pytest.skip( + "set CODEX_RUN_SMOKE=1 to spawn the real Codex binary " + "(no model is called; the backend is stubbed)" + ) + reason = _skip_reason() + if reason is not None: + pytest.skip(reason) + + from google.genai import types + + from veadk import Agent, Runner + from veadk.runtime.codex import proxy as proxy_module + from veadk.runtime.codex import runtime as runtime_module + from veadk.runtime.codex.config import CodexRuntimeConfig + from veadk.runtime.codex.proxy import ResponsesShim + + # Keep every wait bounded: a stalled backend call must not eat the run + # budget, and the shim must fail rather than spin if it cannot bind. + monkeypatch.setenv("CODEX_SHIM_TIMEOUT", "20") + monkeypatch.setenv("CODEX_SHIM_START_TIMEOUT", "10") + + executed: list[str] = [] + + def veadk_smoke_record_marker(marker: str) -> dict: + """Record a marker string. + + Args: + marker: Opaque text to record. + + Returns: + dict: Echo of the recorded marker. + """ + executed.append(marker) + return {"status": "ok", "recorded": marker} + + backend = _StubResponsesBackend() + backend_url = await backend.start() + + # Constructed and started DIRECTLY, never through `get_shim`: that helper + # inserts into the process-global `_SHIMS` cache (a uvicorn server and a + # port that outlive the test). `run_async` calls `get_shim` itself, so the + # lookup -- and only the lookup -- is redirected to this instance. + shim = ResponsesShim(api_base=f"{backend_url}/v1", api_key="veadk-smoke-key") + shims_before = dict(proxy_module._SHIMS) + await shim.start() + assert shim.url and shim.url.startswith("http://127.0.0.1:"), ( + "the shim must bind a real loopback port, not an ASGI transport" + ) + + async def _shim_lookup(api_base: str, api_key: str) -> ResponsesShim: + return shim + + monkeypatch.setattr(runtime_module, "get_shim", _shim_lookup) + + agent = Agent( + name="codex_smoke_agent", + description="Codex end-to-end smoke agent.", + instruction="Call the tool once, then answer in one short sentence.", + runtime="codex", + model_name="veadk-codex-smoke-model", + model_api_base=f"{backend_url}/v1", + model_api_key="veadk-smoke-key", + model_api_key_name="", + tools=[veadk_smoke_record_marker], + codex_runtime_config=CodexRuntimeConfig( + reasoning_effort="minimal", + sandbox="read_only", + network_access=False, + max_tool_iterations=2, + tool_timeout_seconds=20.0, + ), + ) + runner = Runner(agent=agent, app_name="codex_smoke") + + temp_root = Path(tempfile.gettempdir()) + codex_homes_before = _codex_homes(temp_root) + workspace_root = Path(runtime_module._SESSION_WORKSPACE_ROOT) + workspaces_before = set(workspace_root.iterdir()) + # Dot-entries are excluded so a tool cache written by the test session + # itself (`.pytest_cache`) cannot masquerade as a leak; a leaked workspace + # would never be dot-prefixed. + cwd_before = {name for name in os.listdir(os.getcwd()) if name[0] != "."} + + session_id = f"codex-smoke-{uuid.uuid4().hex[:8]}" + user_id = "codex-smoke-user" + await runner.short_term_memory.create_session( + app_name="codex_smoke", user_id=user_id, session_id=session_id + ) + + events: list[Any] = [] + + async def _drive() -> None: + # `aclosing` so that a timeout still throws GeneratorExit into the + # runtime's generator: that is what runs its `finally` (unregister the + # turn, close toolsets, remove CODEX_HOME, reap the subprocess) instead + # of leaving a live Codex process behind for the rest of the session. + stream = runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=types.Content( + role="user", parts=[types.Part(text="Run the smoke tool.")] + ), + ) + async with contextlib.aclosing(stream) as events_stream: + async for event in events_stream: + events.append(event) + + try: + await asyncio.wait_for(_drive(), _RUN_TIMEOUT_SECONDS) + finally: + await shim.stop() + await backend.stop() + + # -- the Codex binary really ran and really talked to the shim --------- + assert backend.requests, ( + "the Codex subprocess never reached the shim: no backend request " + "arrived, so either the binary did not spawn or it rejected the " + "generated config.toml" + ) + + # -- the OS sandbox: established, or documented ------------------------ + assert backend.shell_tool_name is not None, ( + "Codex advertised no function-shaped command tool, so the OS sandbox " + f"was never exercised; tools seen: {_tool_names(backend.requests)}" + ) + probe_output = backend.sandbox_probe_output + assert probe_output is not None, ( + f"Codex never reported a result for its {backend.shell_tool_name!r} " + "call, so the sandboxed command neither succeeded nor failed visibly" + ) + assert _SANDBOX_PROBE_MARKER in probe_output, ( + "the OS sandbox did not establish, or refused to run a trivial " + f"read-only command. Codex reported: {probe_output!r}" + ) + assert _WRITE_DENIED_MARKER in probe_output, ( + "sandbox='read_only' did not actually block a write from inside the " + f"sandbox. Codex reported: {probe_output!r}" + ) + + # -- the ADK tool path over a real socket ------------------------------ + assert executed == [_TOOL_MARKER], ( + f"the ADK tool executor did not run exactly once: {executed!r}" + ) + function_responses = [ + response + for event in events + for response in (event.get_function_responses() or []) + if response.name == _ADK_TOOL_NAME + ] + assert function_responses, ( + "no function_response event reached the Runner for the ADK tool" + ) + + # -- a final text event ------------------------------------------------ + final_texts = [ + part.text + for event in events + if not event.partial and event.content and event.content.parts + for part in event.content.parts + if part.text and not part.thought + ] + assert any(_FINAL_ANSWER in text for text in final_texts), ( + f"the canned final answer never reached the Runner: {final_texts!r}" + ) + + # -- teardown left nothing behind -------------------------------------- + assert shim._turns == {}, ( + f"the turn was not unregistered from the shim: {list(shim._turns)}" + ) + assert dict(proxy_module._SHIMS) == shims_before, ( + "the process-global shim cache was mutated; this test must construct " + "ResponsesShim directly and never call get_shim()" + ) + assert _codex_homes(temp_root) == codex_homes_before, ( + "a CODEX_HOME temp dir survived the turn: " + f"{sorted(_codex_homes(temp_root) - codex_homes_before)}" + ) + new_workspaces = set(workspace_root.iterdir()) - workspaces_before + assert len(new_workspaces) == 1, ( + f"expected exactly one session workspace, got {sorted(new_workspaces)}" + ) + # Session workspaces are deliberately kept for the next turn of the same + # session and reaped later; what must not happen is one escaping the + # process-owned root or landing in the working directory. + workspace = next(iter(new_workspaces)) + assert workspace.parent == workspace_root + assert not (workspace / _WRITE_PROBE_FILE).exists(), ( + "the read-only sandbox let Codex create a file in the workspace" + ) + cwd_after = {name for name in os.listdir(os.getcwd()) if name[0] != "."} + assert cwd_after == cwd_before, ( + "the turn created entries in the working directory: " + f"{sorted(cwd_after - cwd_before)}" + ) + shutil.rmtree(workspace, ignore_errors=True) + + +def _codex_homes(temp_root: Path) -> set[Path]: + """Invocation-scoped CODEX_HOME dirs, excluding the workspace root. + + ``_prepare_codex_home`` and ``_SESSION_WORKSPACE_ROOT`` share the + ``veadk-codex-`` prefix; only the former must be gone after a turn. + """ + return { + path + for path in temp_root.glob("veadk-codex-*") + if path.is_dir() and not path.name.startswith("veadk-codex-workspaces-") + } + + +def _tool_names(requests: list[dict[str, Any]]) -> list[str]: + return sorted( + { + str(tool.get("name")) + for request in requests + for tool in (request.get("tools") or []) + if isinstance(tool, dict) + } + ) diff --git a/tests/runtime/codex/test_codex_sdk_protocol.py b/tests/runtime/codex/test_codex_sdk_protocol.py new file mode 100644 index 000000000..cdafe4748 --- /dev/null +++ b/tests/runtime/codex/test_codex_sdk_protocol.py @@ -0,0 +1,443 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contract tests against the *real* ``openai-codex`` SDK types. + +``translate.py`` dispatches on ``type(payload).__name__`` and ``_tool_call`` +dispatches on a thread item's ``type`` string. Both are string-matched surfaces +against a third-party package, and the rest of the Codex tests hand-roll +same-named fakes -- so an SDK rename or a new notification type is invisible to +every one of them. These tests are the only place the real types are touched. + +They require the ``openai-codex`` extra and therefore **skip on a developer +machine that has not installed it**; CI runs ``uv sync --all-extras`` +(``.github/workflows/unit-tests.yaml``) so they execute there. See +``tests/runtime/codex/README.md``. +""" + +from __future__ import annotations + +import importlib.metadata +import typing + +import pytest + +pytest.importorskip("openai_codex") + +from pydantic import BaseModel, ValidationError # noqa: E402 + +from veadk.runtime.codex import translate # noqa: E402 + +#: Pin from ``pyproject.toml``. Both string-matched surfaces below are only +#: meaningful against a known SDK version. +EXPECTED_SDK_VERSION = "0.1.0b3" + + +def _turn_notification_names() -> set[str]: + """Every notification the SDK can deliver on a *turn* stream. + + Deliberately not "every class ending in ``Notification``": the SDK exports + ~70, most of them thread- or account-scoped and never seen by a turn. The + registry is the SDK's own answer to "what can arrive here". + """ + from openai_codex.generated import notification_registry + + types_ = list(notification_registry.DIRECT_TURN_ID_NOTIFICATION_TYPES) + list( + notification_registry.NESTED_TURN_NOTIFICATION_TYPES + ) + return {t.__name__ for t in types_} + + +def test_dispatch_table_covers_every_sdk_notification() -> None: + """Both directions: an unknown SDK type *and* a stale local entry must fail.""" + sdk_names = _turn_notification_names() + local_names = set(translate._DISPATCH) | translate._EXPLICITLY_IGNORED + + unhandled = sdk_names - local_names + stale = local_names - sdk_names + + assert not unhandled, ( + "openai-codex can deliver notifications this runtime neither handles " + f"nor explicitly ignores: {sorted(unhandled)}. Add a handler to " + "translate._DISPATCH, or record the decision in " + "translate._EXPLICITLY_IGNORED." + ) + assert not stale, ( + "translate.py names notifications the SDK no longer delivers on a turn " + f"stream: {sorted(stale)}. They are dead dispatch entries -- the SDK " + "renamed or removed them, so the real payloads now fall through." + ) + assert not (set(translate._DISPATCH) & translate._EXPLICITLY_IGNORED), ( + "a notification is both handled and explicitly ignored" + ) + + +def test_sdk_pin_matches_pyproject() -> None: + """Both dispatch surfaces are string matches against one pinned version.""" + assert importlib.metadata.version("openai-codex") == EXPECTED_SDK_VERSION + + +def _model_by_name(name: str) -> type[BaseModel]: + from openai_codex.generated import v2_all + + model = getattr(v2_all, name, None) + assert model is not None, f"openai_codex.generated.v2_all has no {name}" + return model + + +def _validate(name: str, payload: dict) -> BaseModel: + """Real pydantic construction, with an actionable failure message.""" + model = _model_by_name(name) + try: + return model.model_validate(payload) + except ValidationError as error: + required = sorted( + field for field, info in model.model_fields.items() if info.is_required() + ) + pytest.fail( + f"{name}.model_validate rejected the payload this suite assumes.\n" + f"required fields: {required}\n{error}" + ) + + +#: Minimal real payloads for the notifications ``translate.py`` reads fields +#: from. These are constructed with ``model_validate`` (never +#: ``model_construct``) so a schema drift fails here rather than silently +#: changing what the runtime observes. +_NOTIFICATION_PAYLOADS: dict[str, dict] = { + # Every turn notification carries `thread_id` alongside the turn scoping, + # and `Turn` itself requires `items` -- both are required by the real + # models, so a payload that omits them never validates. + "TurnStartedNotification": { + "thread_id": "thread-1", + "turn": {"id": "turn-1", "status": "inProgress", "items": []}, + }, + "TurnCompletedNotification": { + "thread_id": "thread-1", + "turn": {"id": "turn-1", "status": "completed", "items": []}, + }, + "AgentMessageDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "delta": "hello", + }, + "ReasoningSummaryTextDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "summary_index": 0, + "delta": "thinking", + }, + "CommandExecutionOutputDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "delta": "stdout chunk", + }, + "ContextCompactedNotification": {"thread_id": "thread-1", "turn_id": "turn-1"}, + "ErrorNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "will_retry": False, + # `TurnError` has no `code`; the structured detail is `codex_error_info`. + "error": {"message": "boom"}, + }, + "FileChangeOutputDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "delta": "patch chunk", + }, + "FileChangePatchUpdatedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "changes": [ + {"path": "/workspace/a.py", "kind": {"type": "add"}, "diff": "+veadk"} + ], + }, + "ItemStartedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "started_at_ms": 0, + "item": {"id": "item-1", "type": "webSearch", "query": "veadk"}, + }, + "ItemCompletedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "completed_at_ms": 1, + "item": {"id": "item-1", "type": "webSearch", "query": "veadk"}, + }, + "ItemGuardianApprovalReviewStartedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "review_id": "review-1", + "started_at_ms": 0, + "target_item_id": "item-1", + "action": {"type": "mcpToolCall", "server": "srv", "tool_name": "tool"}, + "review": {"status": "inProgress"}, + }, + "ItemGuardianApprovalReviewCompletedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "review_id": "review-1", + "started_at_ms": 0, + "completed_at_ms": 1, + "decision_source": "agent", + "target_item_id": "item-1", + "action": {"type": "mcpToolCall", "server": "srv", "tool_name": "tool"}, + "review": {"status": "approved"}, + }, + "McpToolCallProgressNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + # This one names its text field `message`; every other delta member + # names it `delta`, and `_delta_handler` falls back accordingly. + "message": "half way", + }, + "ModelReroutedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "from_model": "gpt-5-codex", + "to_model": "gpt-5", + "reason": "highRiskCyberActivity", + }, + "PlanDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "delta": "plan chunk", + }, + "ReasoningTextDeltaNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "item_id": "item-1", + "content_index": 0, + "delta": "thinking", + }, + "TurnPlanUpdatedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + "plan": [{"step": "do the thing", "status": "inProgress"}], + }, + "ThreadTokenUsageUpdatedNotification": { + "thread_id": "thread-1", + "turn_id": "turn-1", + # `model_context_window` belongs to `ThreadTokenUsage`, not to the + # notification: the SDK nests it beside `last`/`total`, and + # `_on_token_usage` forwards that mapping verbatim. + "token_usage": { + "model_context_window": 128000, + "last": { + "input_tokens": 10, + "cached_input_tokens": 0, + "output_tokens": 4, + "reasoning_output_tokens": 0, + "total_tokens": 14, + }, + "total": { + "input_tokens": 10, + "cached_input_tokens": 0, + "output_tokens": 4, + "reasoning_output_tokens": 0, + "total_tokens": 14, + }, + }, + }, +} + + +def test_every_dispatched_notification_has_a_real_payload() -> None: + """The real-instance test below is only as good as its payload table. + + Without this, adding a handler to ``_DISPATCH`` silently leaves it covered + by hand-rolled fakes alone -- which is the exact blind spot this module + exists to close. + """ + missing = set(translate._DISPATCH) - set(_NOTIFICATION_PAYLOADS) + assert not missing, ( + "these notifications are dispatched but never built as real SDK " + f"instances: {sorted(missing)}. Add a payload to " + "_NOTIFICATION_PAYLOADS." + ) + + +@pytest.mark.parametrize("name", sorted(_NOTIFICATION_PAYLOADS)) +def test_notification_to_events_accepts_real_sdk_instances(name: str) -> None: + """A real SDK instance must reach a handler and translate cleanly. + + Every other Codex test builds a same-named fake with a hand-written + ``model_dump``. If the SDK's real ``model_dump`` nests or renames a field + the handler reads, only this test notices. + """ + assert name in translate._DISPATCH, f"{name} is no longer dispatched" + + payload = _validate(name, _NOTIFICATION_PAYLOADS[name]) + events = translate.notification_to_events(payload, "agent", "inv") + + assert events, f"{name} produced no observable event" + for event in events: + assert event.author == "agent" + assert event.invocation_id == "inv" + assert (event.custom_metadata or {}).get("codex_event_type") + + +def test_turn_completed_payload_exposes_turn_id() -> None: + """The nested ``turn.id`` must survive both attribute access and the dump. + + ``TurnCompletedNotification`` carries no ``turn_id`` of its own -- the id + is nested under ``turn`` -- and ``_on_turn_completed`` reads it out of the + dumped mapping to stamp ``custom_metadata["turn_id"]``. The SDK's own + ``notification_turn_id`` resolves it by attribute, so assert both paths. + """ + payload = _validate( + "TurnCompletedNotification", _NOTIFICATION_PAYLOADS["TurnCompletedNotification"] + ) + + assert payload.turn.id == "turn-1" + event = translate.notification_to_events(payload, "agent", "inv")[0] + assert event.custom_metadata["turn_id"] == "turn-1" + + +# ------------------------------------------------ thread item discriminators + + +def _item_model_for(discriminator: str) -> type[BaseModel]: + """Find the *thread item* model whose ``type`` literal is ``discriminator``. + + Scoped to the SDK's own ``ThreadItem`` union rather than to every model in + ``v2_all``, because the discriminator strings are not globally unique: + ``mcpToolCall`` is carried by both ``McpToolCallThreadItem`` and + ``McpToolCallGuardianApprovalReviewAction``. Only a thread item is ever + handed to ``_tool_call``, and scanning the whole module picked the + approval-review model by alphabetical accident. + + Looked up by discriminator rather than by class name so an SDK *rename* + still resolves; a removed or renamed *discriminator* -- which is what + ``_tool_call`` actually matches on -- fails loudly. + """ + from openai_codex.generated import v2_all + + union = typing.get_args(v2_all.ThreadItem.model_fields["root"].annotation) + matches: list[type[BaseModel]] = [] + for candidate in union: + field = candidate.model_fields.get("type") + if field is None: + continue + literals = typing.get_args(field.annotation) + if discriminator in literals or field.default == discriminator: + matches.append(candidate) + assert matches, ( + f"no openai-codex thread item carries type={discriminator!r}; " + "translate._tool_call still matches that string and would now return " + "None for every such item, silently dropping the tool call" + ) + assert len(matches) == 1, ( + f"type={discriminator!r} is ambiguous within the ThreadItem union: " + f"{[m.__name__ for m in matches]}" + ) + return matches[0] + + +@pytest.mark.parametrize( + ("discriminator", "expected_tool_name", "payload"), + [ + pytest.param( + "commandExecution", + "exec_command", + { + "id": "item-1", + "type": "commandExecution", + "command": "ls", + # Required by CommandExecutionThreadItem: the SDK parses every + # command into a structured action list. + "command_actions": [], + "cwd": "/workspace", + "aggregated_output": "a\nb\n", + "exit_code": 0, + "status": "completed", + }, + id="commandExecution", + ), + pytest.param( + "mcpToolCall", + "srv.tool", + { + "id": "item-2", + "type": "mcpToolCall", + "server": "srv", + "tool": "tool", + "arguments": "{}", + "status": "completed", + }, + id="mcpToolCall", + ), + pytest.param( + "dynamicToolCall", + "ns.tool", + { + "id": "item-3", + "type": "dynamicToolCall", + "namespace": "ns", + "tool": "tool", + "arguments": "{}", + "status": "completed", + }, + id="dynamicToolCall", + ), + pytest.param( + "fileChange", + "apply_patch", + { + "id": "item-4", + "type": "fileChange", + "changes": [], + "status": "completed", + }, + id="fileChange", + ), + pytest.param( + "webSearch", + "web_search", + {"id": "item-5", "type": "webSearch", "query": "veadk"}, + id="webSearch", + ), + ], +) +def test_thread_item_discriminators_match_sdk( + discriminator: str, expected_tool_name: str, payload: dict +) -> None: + """``_tool_call`` is a second string-matched surface, previously untested. + + It maps a thread item's ``type`` onto an ADK tool name. A discriminator + rename makes it return ``None``, which drops the ``function_call`` / + ``function_response`` pair for that tool entirely -- with no error. + """ + model = _item_model_for(discriminator) + try: + instance = model.model_validate(payload) + except ValidationError as error: + pytest.fail(f"{model.__name__}.model_validate rejected {payload!r}:\n{error}") + + dumped = instance.model_dump() + assert dumped.get("type") == discriminator, dumped + + call = translate._tool_call(dumped) + assert call is not None, ( + f"_tool_call returned None for a real {discriminator} item; the tool " + "call and its result would be silently dropped" + ) + assert call[0] == expected_tool_name, call diff --git a/tests/runtime/codex/test_codex_shim_rounds.py b/tests/runtime/codex/test_codex_shim_rounds.py new file mode 100644 index 000000000..c7225e5ac --- /dev/null +++ b/tests/runtime/codex/test_codex_shim_rounds.py @@ -0,0 +1,1393 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-round tests for the Responses shim. + +These were split out of the 936-line ``test_codex_runtime.py``, whose size was +a proximate cause of the coverage gap they close: every shim test in that file +sent exactly one POST per turn with ``stream: False``, so the ~113-line +``_synth_sse`` had zero coverage and the worst bug -- tool history lost between +two requests under one token -- was a shape no test could express. + +Every test here builds :class:`ResponsesShim` **directly** over +``httpx.ASGITransport``, so no uvicorn server is started and no port is bound, +which is what makes the file safe under ``pytest -n 16``. The one test that has +to exercise ``get_shim`` itself (the cache is what it is testing) swaps the +process-global ``_SHIMS``/``_RETIRED`` for empty ones and restores them in a +``finally``, and stubs ``start()`` so nothing binds either; the autouse fixture +below re-checks that from the outside. +""" + +from __future__ import annotations + +import asyncio +import gc +import json +import threading +import time +from collections import OrderedDict + +import httpx +import pytest + +from veadk.runtime.codex import proxy as proxy_module +from veadk.runtime.codex.proxy import ResponsesShim + + +@pytest.fixture(autouse=True) +def _shim_cache_is_untouched(): + """Fail loudly if a test in this file ever starts a real shim server. + + ``get_shim`` binds a port and leaks a uvicorn task into the process-global + cache; nothing here may do that. A test that must call ``get_shim`` swaps + the global out and back itself (see ``_isolated_shim_cache``), so this still + holds for it. + """ + before = dict(proxy_module._SHIMS) + yield + assert dict(proxy_module._SHIMS) == before, ( + "a test in this file mutated the process-global shim cache; construct " + "ResponsesShim directly instead of calling get_shim()" + ) + + +#: Distinguishes "the key is absent" from "the key is present and empty", which +#: select different branches of the shim's tool-advertisement logic. +_UNSET = object() + + +def _client(shim: ResponsesShim) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.ASGITransport(app=shim._app), base_url="http://shim" + ) + + +def _message(text: str, role: str = "user") -> dict: + return {"type": "message", "role": role, "content": [{"text": text}]} + + +def _text_response(text: str, response_id: str = "resp") -> dict: + return { + "id": response_id, + "model": "model", + "output": [ + { + "id": f"msg-{response_id}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text}], + } + ], + } + + +def _tool_response(name: str, response_id: str, call_id: str) -> dict: + return { + "id": response_id, + "model": "model", + "output": [ + { + "id": f"fc-{response_id}", + "call_id": call_id, + "type": "function_call", + "name": name, + "arguments": "{}", + "status": "completed", + } + ], + } + + +def _sse_frames(body: str) -> list[tuple[str, dict]]: + """Parse a ``text/event-stream`` body into ``(event name, data)`` pairs.""" + frames: list[tuple[str, dict]] = [] + for raw in body.split("\n\n"): + if not raw.strip(): + continue + name = None + data = None + for line in raw.splitlines(): + if line.startswith("event: "): + name = line[len("event: ") :] + elif line.startswith("data: "): + data = json.loads(line[len("data: ") :]) + assert name is not None, f"SSE frame without an event line: {raw!r}" + assert data is not None, f"SSE frame without a data line: {raw!r}" + frames.append((name, data)) + return frames + + +# ---------------------------------------------------- the multi-round blocker + + +@pytest.mark.asyncio +async def test_two_requests_under_one_token_replay_the_tool_transcript( + monkeypatch, +) -> None: + """The blocker: the second request must carry round one's tool pair. + + Codex rebuilds ``input`` from its own thread on every request and never saw + the shim-executed ``function_call``/``function_call_output`` pair, because + those are deliberately not streamed to it. Without a replay the model sees a + conversation in which it never called the tool, and re-issues the call -- + re-running its side effects. Every previous shim test sent exactly one POST + per turn, so this shape could not be expressed at all. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + executed: list[str] = [] + seen: list[list[dict]] = [] + + async def executor(args, call_id): + executed.append(call_id) + return json.dumps({"ok": True}) + + token = shim.register_turn( + [{"type": "function", "name": "record", "parameters": {}}], + {"record": executor}, + ) + + async def backend(**kwargs): + seen.append(json.loads(json.dumps(kwargs["input"]))) + has_output = any( + item.get("type") == "function_call_output" for item in kwargs["input"] + ) + if has_output: + return _text_response("all done", f"final-{len(seen)}") + return _tool_response("record", f"tool-{len(seen)}", "call-1") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + headers = {"Authorization": f"Bearer {token}"} + first = await client.post( + "/v1/responses", + headers=headers, + json={"model": "model", "stream": False, "input": [_message("go")]}, + ) + # Codex's own second request: a fresh `input` rebuilt from its thread, + # with no knowledge of the tool the shim ran on its behalf. Codex + # appends its *own* items (here a native tool round) and re-sends the + # same user message -- it does not add a new user turn. That shape + # matters: a trailing user message is what a compaction pass looks + # like, and the agent-turn gate rejects those on purpose. + second = await client.post( + "/v1/responses", + headers=headers, + json={ + "model": "model", + "stream": False, + "input": [ + _message("go"), + { + "type": "function_call", + "call_id": "shell-1", + "name": "exec_command", + "arguments": "{}", + "status": "completed", + }, + { + "type": "function_call_output", + "call_id": "shell-1", + "output": "ok", + }, + ], + }, + ) + + assert first.status_code == 200 + assert second.status_code == 200 + assert executed == ["call-1"], "the tool must run exactly once for the turn" + + # The replayed ADK pair must be there. Codex's own `shell-1` pair is there + # too -- it is part of the history Codex rebuilt -- so this asserts on the + # ADK call id rather than on the request containing nothing else. + kinds = [ + (item.get("type"), item.get("call_id")) + for item in seen[-1] + if item.get("type") in ("function_call", "function_call_output") + ] + assert kinds.count(("function_call", "call-1")) == 1, seen[-1] + assert kinds.count(("function_call_output", "call-1")) == 1, seen[-1] + # Order matters to the chat bridge: each call must be immediately followed + # by its own result, or litellm emits an `assistant(tool_calls)` with no + # matching `tool` message and the backend rejects the request. + call_index = kinds.index(("function_call", "call-1")) + assert kinds[call_index + 1] == ("function_call_output", "call-1"), kinds + + +@pytest.mark.asyncio +async def test_turn_state_is_not_shared_across_tokens(monkeypatch) -> None: + """One turn's tool transcript must never leak into another turn's request.""" + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + + async def executor(args, call_id): + return json.dumps({"ok": True}) + + specs = [{"type": "function", "name": "record", "parameters": {}}] + token_a = shim.register_turn(specs, {"record": executor}) + token_b = shim.register_turn(specs, {"record": executor}) + seen: dict[str, list[list[dict]]] = {"a": [], "b": []} + which = {"value": "a"} + + async def backend(**kwargs): + seen[which["value"]].append(json.loads(json.dumps(kwargs["input"]))) + if any(i.get("type") == "function_call_output" for i in kwargs["input"]): + return _text_response("done") + return _tool_response("record", "tool", f"call-{which['value']}") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + body = {"model": "model", "stream": False, "input": [_message("go")]} + which["value"] = "a" + await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token_a}"}, + json=json.loads(json.dumps(body)), + ) + which["value"] = "b" + await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token_b}"}, + json=json.loads(json.dumps(body)), + ) + + replayed_into_b = [ + item + for request in seen["b"] + for item in request + if item.get("call_id") == "call-a" + ] + assert replayed_into_b == [], replayed_into_b + + +# ------------------------------------------------------- streamed SSE shape + + +@pytest.mark.asyncio +async def test_streamed_turn_emits_canonical_dense_sse_sequence(monkeypatch) -> None: + """``stream: True`` must produce Codex's canonical event order. + + Every pre-existing shim test passed ``stream: False``, so ``_synth_sse`` + -- the ~113 lines that decide what Codex actually acts on -- had no + coverage at all. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + token = shim.register_turn([], {}) + + async def backend(**kwargs): + return _text_response("hello there") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": True, "input": [_message("go")]}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + + frames = _sse_frames(response.text) + names = [name for name, _ in frames] + assert names[0] == "response.created" + assert names[1] == "response.in_progress" + assert names[-1] == "response.completed" + assert "response.output_item.added" in names + assert "response.output_text.delta" in names + assert "response.output_item.done" in names + # The `event:` line must match the payload's own `type`, or Codex's parser + # dispatches on one thing and reads another. + assert all(name == data["type"] for name, data in frames) + # Dense, gapless, strictly increasing: Codex rejects a stream with holes. + assert [data["sequence_number"] for _, data in frames] == list(range(len(frames))) + + +@pytest.mark.asyncio +async def test_streamed_function_call_survives_synthesis(monkeypatch) -> None: + """A dropped tool call would silently end the turn at the preamble. + + The tool call is what drives Codex's agentic loop. If ``_synth_sse`` filters + it out, Codex sees a turn that finished after saying "let me look..." -- no + error, no retry, just a wrong answer. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + token = shim.register_turn([], {}) # no executor: the call passes through + + async def backend(**kwargs): + return { + "id": "resp", + "model": "model", + "output": [ + { + "id": "msg", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "let me look..."}], + }, + { + "id": "fc", + "call_id": "call-1", + "type": "function_call", + "name": "shell", + "arguments": '{"command":"ls"}', + "status": "completed", + }, + ], + } + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": True, "input": [_message("go")]}, + ) + + frames = _sse_frames(response.text) + by_name = {name: data for name, data in frames} + assert "response.function_call_arguments.delta" in by_name + assert by_name["response.function_call_arguments.done"]["arguments"] == ( + '{"command":"ls"}' + ) + done_items = [data["item"] for name, data in frames if name.endswith("item.done")] + assert [item["type"] for item in done_items] == ["message", "function_call"] + assert done_items[1]["name"] == "shell" + + +@pytest.mark.asyncio +async def test_streamed_reasoning_summary_events_are_emitted(monkeypatch) -> None: + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + token = shim.register_turn([], {}) + + async def backend(**kwargs): + return { + "id": "resp", + "model": "model", + "output": [ + { + "id": "rs", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "thinking"}], + }, + { + "id": "msg", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "answer"}], + }, + ], + } + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": True, "input": [_message("go")]}, + ) + + names = [name for name, _ in _sse_frames(response.text)] + assert "response.reasoning_summary_part.added" in names + assert "response.reasoning_summary_text.delta" in names + assert "response.reasoning_summary_text.done" in names + assert "response.reasoning_summary_part.done" in names + + +@pytest.mark.asyncio +async def test_response_completed_output_equals_the_streamed_items( + monkeypatch, +) -> None: + """Pins the silent trimming: the terminal payload must match what streamed. + + ``_synth_sse`` streams only ``message``/``reasoning``/``function_call`` + items and rewrites ``response.completed.output`` to match. If the two ever + disagree, Codex's reconciliation sees items it never received (or loses ones + it did) with no error anywhere. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + token = shim.register_turn([], {}) + + async def backend(**kwargs): + return { + "id": "resp", + "model": "model", + "output": [ + { + "id": "msg", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "answer"}], + }, + # Not one of the streamed kinds: must be trimmed from both. + {"id": "misc", "type": "web_search_call", "status": "completed"}, + ], + } + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": True, "input": [_message("go")]}, + ) + + frames = _sse_frames(response.text) + streamed = [data["item"] for name, data in frames if name.endswith("item.done")] + completed = frames[-1][1]["response"] + assert frames[-1][0] == "response.completed" + assert completed["status"] == "completed" + assert completed["output"] == streamed + assert [item["type"] for item in completed["output"]] == ["message"] + + +# ------------------------------------------------------ Ark status backfill + + +@pytest.mark.asyncio +async def test_assistant_messages_are_backfilled_with_a_status(monkeypatch) -> None: + """Ark's Responses API rejects an assistant message with no ``status``. + + Codex replays prior assistant messages without one, so a turn with a model + preamble followed by a tool call used to die on ``MissingParameter: + input.status``. This path had zero tests. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + token = shim.register_turn([], {}) + seen: list[list[dict]] = [] + + async def backend(**kwargs): + seen.append(json.loads(json.dumps(kwargs["input"]))) + return _text_response("ok") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "model": "model", + "stream": False, + "input": [ + _message("go"), + _message("let me look...", role="assistant"), + { + "type": "message", + "role": "assistant", + "status": "incomplete", + "content": [{"text": "partially done"}], + }, + ], + }, + ) + + forwarded = seen[0] + assert forwarded[1]["status"] == "completed", "missing status was not backfilled" + assert forwarded[2]["status"] == "incomplete", "an explicit status was overwritten" + assert "status" not in forwarded[0], "a user message must not be given a status" + + +@pytest.mark.asyncio +async def test_shim_routes_concurrent_turns_to_their_own_executors( + monkeypatch, +) -> None: + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + calls: list[tuple[str, str]] = [] + + async def executor_a(args, call_id): + await asyncio.sleep(0.01) + calls.append(("a", call_id)) + return json.dumps({"owner": "a"}) + + async def executor_b(args, call_id): + calls.append(("b", call_id)) + return json.dumps({"owner": "b"}) + + token_a = shim.register_turn( + [{"type": "function", "name": "tool_a", "parameters": {}}], + {"tool_a": executor_a}, + ) + token_b = shim.register_turn( + [{"type": "function", "name": "tool_b", "parameters": {}}], + {"tool_b": executor_b}, + ) + + async def fake_aresponses(**kwargs): + conversation = kwargs["input"] + if any(item.get("type") == "function_call_output" for item in conversation): + return { + "id": "resp-final", + "model": "model", + "output": [ + { + "id": "msg", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done"}], + } + ], + } + tool = next( + item for item in kwargs["tools"] if item["name"] in {"tool_a", "tool_b"} + ) + suffix = tool["name"][-1] + return { + "id": f"resp-{suffix}", + "model": "model", + "output": [ + { + "id": f"fc-{suffix}", + "call_id": f"call-{suffix}", + "type": "function_call", + "name": tool["name"], + "arguments": "{}", + "status": "completed", + } + ], + } + + monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", fake_aresponses) + transport = httpx.ASGITransport(app=shim._app) + async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: + body = { + "model": "model", + "stream": False, + "input": [{"type": "message", "role": "user", "content": "go"}], + } + response_a, response_b = await asyncio.gather( + client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token_a}"}, + json=body, + ), + client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token_b}"}, + json=body, + ), + ) + + assert response_a.status_code == 200 + assert response_b.status_code == 200 + assert sorted(calls) == [("a", "call-a"), ("b", "call-b")] + + +@pytest.mark.asyncio +async def test_shim_rejects_unknown_invocation_token() -> None: + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + transport = httpx.ASGITransport(app=shim._app) + async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": "Bearer unknown"}, + json={"model": "model", "input": []}, + ) + assert response.status_code == 401 + + +@pytest.mark.asyncio +async def test_shim_reports_tool_iteration_budget_instead_of_dropping_call( + monkeypatch, +) -> None: + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + + async def executor(args, call_id): + return "{}" + + token = shim.register_turn( + [{"type": "function", "name": "loop", "parameters": {}}], + {"loop": executor}, + max_tool_iterations=1, + ) + + async def always_calls_tool(**kwargs): + return { + "id": "resp", + "model": "model", + "output": [ + { + "id": "fc", + "call_id": "call-loop", + "type": "function_call", + "name": "loop", + "arguments": "{}", + "status": "completed", + } + ], + } + + monkeypatch.setattr( + "veadk.runtime.codex.proxy.litellm.aresponses", always_calls_tool + ) + transport = httpx.ASGITransport(app=shim._app) + async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "model": "model", + "input": [{"type": "message", "role": "user", "content": "go"}], + }, + ) + + assert response.status_code == 409 + assert response.json()["error"]["type"] == "tool_iteration_limit" + + +@pytest.mark.asyncio +async def test_shim_rejects_invalid_tool_json_without_calling_executor( + monkeypatch, +) -> None: + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + called = False + + async def executor(args, call_id): + nonlocal called + called = True + return "{}" + + token = shim.register_turn( + [{"type": "function", "name": "parse", "parameters": {}}], + {"parse": executor}, + ) + + async def fake_aresponses(**kwargs): + if any(item.get("type") == "function_call_output" for item in kwargs["input"]): + return { + "id": "final", + "model": "model", + "output": [ + { + "id": "msg", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "handled"}], + } + ], + } + return { + "id": "tool", + "model": "model", + "output": [ + { + "id": "fc", + "call_id": "call-invalid", + "type": "function_call", + "name": "parse", + "arguments": "{not-json", + "status": "completed", + } + ], + } + + monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", fake_aresponses) + transport = httpx.ASGITransport(app=shim._app) + async with httpx.AsyncClient(transport=transport, base_url="http://shim") as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "input": []}, + ) + + assert response.status_code == 200 + assert called is False + + +@pytest.mark.asyncio +async def test_shim_returns_the_turn_total_token_usage(monkeypatch) -> None: + """The shim's internal tool loop must not throw away tokens it spent. + + Codex sees exactly one request per turn here: the shim executes the agent's + tools itself and returns only the final, tool-free response. Every + intermediate backend response -- and the ``usage`` block it carries -- is + discarded, so the tokens the tool rounds cost never reach Codex, and + therefore never reach ``usage_metadata``, the ``call_llm`` span, portal + metrics or the frontend token counter. The turn silently under-reports. + + ``tests/runtime/differential/test_runtime_parity.py::…[usage_accounting]`` + is the same gap observed end to end: ADK reports 18/8, Codex reports 7/3. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + + async def executor(args, call_id): + return json.dumps({"ok": True}) + + token = shim.register_turn( + [{"type": "function", "name": "record", "parameters": {}}], + {"record": executor}, + ) + rounds = iter( + [ + { + **_tool_response("record", "tool", "call-1"), + "usage": {"input_tokens": 11, "output_tokens": 5, "total_tokens": 16}, + }, + { + **_text_response("all done"), + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + }, + ] + ) + + async def backend(**kwargs): + return next(rounds) + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": False, "input": [_message("go")]}, + ) + + usage = response.json().get("usage") or {} + assert usage.get("input_tokens") == 18, ( + f"the tool round's 11 input tokens were dropped: {usage}" + ) + assert usage.get("output_tokens") == 8, ( + f"the tool round's 5 output tokens were dropped: {usage}" + ) + assert usage.get("total_tokens") == 26, usage + + +# ------------------------------------------- compaction / review contamination +# +# Codex reuses one provider block -- and therefore one bearer token -- for work +# that is not the agent turn. Verified against the Codex source at +# `codex-rs/core/src/compact.rs::run_compact_task_inner_impl`, which builds its +# prompt with an empty `tools` list and sends it through `turn_context.provider`, +# and `codex-rs/core/src/session/review.rs::spawn_review_thread`, which clones +# the parent turn's provider (and `runtime.py` points `review_model` at the same +# model, so there is exactly one provider block). +# +# The shim must not treat those requests as the agent turn: advertising the +# agent's ADK tools to a summarizer, or replaying the turn's tool transcript +# into it, invites a `function_call` in the reply -- and the shim would then +# execute the real tool a second time, the same duplicated side effect the +# multi-round blocker above is about. + + +def _compaction_body(*, tools: object = _UNSET) -> dict: + """A request shaped like Codex's compaction pass, not like an agent turn. + + ``tools`` defaults to being absent entirely (the `elif turn_context.specs` + branch); pass ``[]`` for the shape `compact.rs` actually sends (the + `isinstance(tools, list)` branch). + """ + body = { + "model": "model", + "stream": False, + "instructions": "You are summarizing a conversation.", + "input": [ + _message("Earlier we discussed the deployment."), + _message("Summarize the conversation so far.", role="user"), + ], + "store": False, + } + if tools is not _UNSET: + body["tools"] = tools + return body + + +async def _register_turn_with_history(shim, executed, monkeypatch): + """Run one real agent round so the turn has a tool transcript to leak.""" + + async def executor(args, call_id): + executed.append(call_id) + return json.dumps({"ok": True}) + + token = shim.register_turn( + [ + { + "type": "function", + "name": "record_fact", + "parameters": {"type": "object", "properties": {}}, + } + ], + {"record_fact": executor}, + ) + + rounds = iter( + [ + _tool_response("record_fact", "resp-1", "call-1"), + _text_response("done", "resp-2"), + ] + ) + + async def backend(**kwargs): + return next(rounds) + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + async with _client(shim) as client: + await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "model", "stream": False, "input": [_message("go")]}, + ) + assert executed == ["call-1"], "setup failed: the agent round ran no tool" + return token + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "shape"), + [ + pytest.param([], "empty-list", id="tools_empty_list"), + pytest.param(_UNSET, "absent", id="tools_absent"), + ], +) +async def test_compaction_request_gets_no_adk_tools_and_no_transcript_replay( + monkeypatch, tools, shape +) -> None: + """A non-agent-turn request on a live token must be left alone. + + Both shim branches fire on this shape today: ``tools: []`` hits the + ``isinstance(tools, list)`` branch and an absent ``tools`` hits the + ``elif turn_context.specs`` branch, so compaction is handed the agent's ADK + tools either way, plus the turn's ``function_call``/``function_call_output`` + pair. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + executed: list[str] = [] + token = await _register_turn_with_history(shim, executed, monkeypatch) + + seen: list[dict] = [] + + async def backend(**kwargs): + seen.append(kwargs) + return _text_response("a summary", "resp-compact") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json=_compaction_body(tools=tools), + ) + + assert response.status_code == 200, response.text + assert len(seen) == 1, seen + sent = seen[0] + + advertised = { + t.get("name") for t in (sent.get("tools") or []) if isinstance(t, dict) + } + assert "record_fact" not in advertised, ( + f"the agent's ADK tool was advertised to a compaction pass ({shape} " + f"tools): {sent.get('tools')!r}. The summarizer can now emit a " + "function_call for it, which the shim would execute for real." + ) + + replayed = [ + item + for item in (sent.get("input") or []) + if isinstance(item, dict) + and item.get("type") in ("function_call", "function_call_output") + ] + assert replayed == [], ( + f"the agent turn's tool transcript leaked into a compaction pass " + f"({shape} tools): {replayed!r}" + ) + + +@pytest.mark.asyncio +async def test_compaction_request_never_executes_an_adk_tool(monkeypatch) -> None: + """The consequence, asserted directly: no second side effect. + + Even if the summarizer's reply contains a ``function_call`` naming an ADK + tool, the shim must not run it. This is the assertion that would have caught + the bug regardless of how tool advertisement is gated. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + executed: list[str] = [] + token = await _register_turn_with_history(shim, executed, monkeypatch) + assert executed == ["call-1"] + + calls = 0 + + async def backend(**kwargs): + nonlocal calls + calls += 1 + # A summarizer that (however unwisely) asks for the agent's tool. + if calls == 1: + return _tool_response("record_fact", "resp-compact", "call-compact") + return _text_response("a summary", "resp-compact-2") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json=_compaction_body(tools=[]), + ) + + assert executed == ["call-1"], ( + "a compaction pass re-executed the agent's tool: the real side effect " + f"ran twice (call ids: {executed})" + ) + + +@pytest.mark.asyncio +async def test_degraded_gate_rejects_a_compaction_shaped_request(monkeypatch) -> None: + """The marker-less fallback must still fail closed on a compaction pass. + + When the turn marker never reaches the model request, the gate falls back to + matching the first request's user texts. Matching *any* remembered text + would admit compaction, which re-sends the whole history and therefore + always carries the turn's opening message -- and a summarizer that emits a + ``function_call`` would run a real ADK tool a second time. Only the *last* + user message is anchored, so compaction's appended instruction fails it. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + executed: list[str] = [] + seen: list[dict] = [] + + async def executor(args, call_id): + executed.append(call_id) + return json.dumps({"ok": True}) + + token = shim.register_turn( + [{"type": "function", "name": "record", "parameters": {}}], + {"record": executor}, + ) + + async def backend(**kwargs): + seen.append(json.loads(json.dumps(kwargs))) + return _text_response("summary", f"resp-{len(seen)}") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", backend) + + async with _client(shim) as client: + headers = {"Authorization": f"Bearer {token}"} + # First request establishes the anchors (no marker: degraded path). + await client.post( + "/v1/responses", + headers=headers, + json={"model": "model", "stream": False, "input": [_message("go")]}, + ) + # Compaction: whole history re-sent, a summarization instruction + # appended as a trailing user message, and an empty tools list. + await client.post( + "/v1/responses", + headers=headers, + json={ + "model": "model", + "stream": False, + "tools": [], + "input": [_message("go"), _message("Summarize the conversation.")], + }, + ) + + assert executed == [], "no ADK tool may run for a compaction pass" + compaction = seen[-1] + assert not compaction.get("tools"), ( + "the agent's ADK tools must not be advertised to the summarizer, or it " + f"can call them: {compaction.get('tools')!r}" + ) + replayed = [ + item + for item in compaction["input"] + if item.get("type") in ("function_call", "function_call_output") + ] + assert replayed == [], f"tool transcript leaked into compaction: {replayed}" + + +def test_extra_body_drops_keys_the_responses_transport_rejects() -> None: + """VeADK's default caching block must not be forwarded to a Responses call. + + Codex always sends the Responses ``instructions`` field, and Ark rejects + prompt caching alongside it ("caching is not supported for instructions"), + so forwarding ``DEFAULT_MODEL_EXTRA_CONFIG`` verbatim 400s *every* turn. + Forwarding ``model_extra_config`` at all is new; before it, the whole body + was dropped and this could not happen. Attribution headers -- the valuable + half -- must still go through, and a user's own body keys must be untouched. + """ + from veadk.consts import DEFAULT_MODEL_EXTRA_CONFIG + from veadk.runtime.codex.proxy import _split_model_extra_config + + headers, body = _split_model_extra_config(DEFAULT_MODEL_EXTRA_CONFIG) + assert "caching" not in body, body + assert "expire_at" not in body, body + assert headers["veadk-source"] == "veadk" + assert "x-is-encrypted" in headers + + _, user_body = _split_model_extra_config( + {"extra_body": {"thinking": {"type": "disabled"}, "caching": {"type": "on"}}} + ) + assert user_body == {"thinking": {"type": "disabled"}}, user_body + + +@pytest.mark.asyncio +async def test_backend_error_is_recorded_so_the_turn_cannot_finish_silently( + monkeypatch, +) -> None: + """A rejected backend request must not read as a completed turn. + + Codex treats a rejected request as the end of its turn and returns whatever + it already had, so a 4xx that is only logged produces `status=completed`, a + half-finished workspace and a plausible-sounding summary -- a silently wrong + answer. Recording it on the turn state is what lets the runtime re-raise + once the stream ends. The message must also reach the log and the client + without the backend credential in it. + """ + from litellm import exceptions as litellm_exceptions + + shim = ResponsesShim("https://backend.invalid/v1", "sk-secret-key") + token = shim.register_turn([], {}, invocation_id="inv-err") + + async def boom(**kwargs): + raise litellm_exceptions.BadRequestError( + message="input[3].reasoning: not supported. key=sk-secret-key", + model="m", + llm_provider="openai", + ) + + monkeypatch.setattr(proxy_module.litellm, "aresponses", boom) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={"model": "m", "input": [_message("hi")]}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["type"] == "invalid_request_error" + assert "sk-secret-key" not in json.dumps(response.json()) + recorded = shim.turn_error(token) + assert isinstance(recorded, litellm_exceptions.BadRequestError), recorded + + +@pytest.mark.asyncio +async def test_reasoning_rejection_retries_once_without_reasoning_items( + monkeypatch, +) -> None: + """A model that refuses replayed reasoning items must still be usable. + + After its first tool round Codex replays its own ``reasoning`` items in + ``input``. Ark refuses them per-model (``doubao-seed-1-6``), which killed + the turn mid-investigation and made that model family unusable with this + runtime. The retry drops reasoning items only in response to that specific + refusal -- never pre-emptively, since for backends that accept them they + carry the chain of thought across tool rounds. + """ + from litellm import exceptions as litellm_exceptions + + conversation = [ + _message("go"), + {"type": "reasoning", "summary": [{"text": "thinking"}]}, + {"type": "function_call", "call_id": "c1", "name": "t", "arguments": "{}"}, + ] + seen: list[list[dict]] = [] + + async def refuses_reasoning(**kwargs): + seen.append(kwargs["input"]) + if any(item.get("type") == "reasoning" for item in kwargs["input"]): + raise litellm_exceptions.BadRequestError( + message="input[1].reasoning is not supported for model", + model="doubao-seed-1-6", + llm_provider="openai", + ) + return {"id": "r", "output": [], "usage": {}} + + monkeypatch.setattr(proxy_module.litellm, "aresponses", refuses_reasoning) + result = await proxy_module._call_backend_tolerating_reasoning( + {"model": "m", "input": list(conversation)} + ) + assert result["id"] == "r" + assert len(seen) == 2, "exactly one retry" + assert not any(item.get("type") == "reasoning" for item in seen[-1]) + assert [item["type"] for item in seen[-1]] == ["message", "function_call"] + + # An unrelated failure must not trigger the strip, or a real error would be + # masked by a second identical request. + seen.clear() + + async def unrelated(**kwargs): + seen.append(kwargs["input"]) + raise litellm_exceptions.BadRequestError( + message="quota exceeded", model="m", llm_provider="openai" + ) + + monkeypatch.setattr(proxy_module.litellm, "aresponses", unrelated) + with pytest.raises(litellm_exceptions.BadRequestError): + await proxy_module._call_backend_tolerating_reasoning( + {"model": "m", "input": list(conversation)} + ) + assert len(seen) == 1, "no retry for an unrelated error" + + +# ------------------------------------------- the get_shim -> register_turn gap + + +def test_a_reservation_outlives_its_deadline_while_the_lease_is_held( + monkeypatch, +) -> None: + """A slow setup must not lose the shim it is about to register a turn on. + + ``get_shim`` returns long before ``register_turn``: in between the runtime + prepares a workspace, reaps stale ones, prepares a ``CODEX_HOME``, syncs + skills and builds its toolsets -- which connects MCP servers. Any constant + deadline is a guess about how long that takes, and past it the shim is + evictable again: with the cache over ``CODEX_SHIM_CACHE_MAX`` it is stopped, + the turn registers on a corpse, and Codex spends the whole turn pointed at a + dead URL. + + So the deadline is only a floor. What actually holds the reservation open is + the lease the caller is holding -- tracked weakly, so it needs no release + call that an exception or an abandoned async generator could skip. Here the + floor is set to 50ms and then deliberately overrun. + """ + monkeypatch.setenv("CODEX_SHIM_RESERVE_SECONDS", "0.05") + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + + lease = shim.reserve() + assert shim.busy + + time.sleep(0.12) # well past the floor: setup is still running + assert shim.busy, ( + "the reservation lapsed while its caller was still in setup; the LRU " + "may now stop this shim out from under the turn about to register" + ) + + # Dropping the lease is the release -- no call to miss on any exit path. + del lease + gc.collect() + assert not shim.busy, ( + "a dropped lease must release the shim (past the floor), or a caller " + "that crashed mid-setup would pin it in the cache forever" + ) + + +def test_a_dropped_lease_is_still_covered_by_the_deadline_floor(monkeypatch) -> None: + """The floor still protects a caller that kept only the URL. + + ``get_shim_url`` returns a string and drops the lease immediately, and an + embedder may do the same. Releasing on the spot would hand those callers a + URL to a shim that is evictable the moment they look away, so the + ``CODEX_SHIM_RESERVE_SECONDS`` window remains underneath the lease rather + than being replaced by it. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + shim.reserve() # lease dropped immediately, as `get_shim_url` does + gc.collect() + assert shim.busy, "the reservation floor must survive a dropped lease" + + +def test_register_turn_without_a_reservation_cannot_consume_someone_elses() -> None: + """A direct ``register_turn`` must not cancel another caller's protection. + + Registering without reserving first is supported (tests, embedders). It used + to pop ``_reservations[0]`` -- the *oldest* reservation, belonging to + whichever other caller happened to be in setup -- so an unrelated turn + starting on the same shim silently re-opened that caller's eviction window. + Reservations are identified now: a caller consumes its own or nothing. + """ + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + + # Caller A: mid-setup, holding its lease. + lease_a = shim.reserve() + reservation_a = lease_a._reservation_id + + # Caller B: registers directly, having never reserved. + token_b = shim.register_turn([], {}) + shim.unregister_turn(token_b) + + assert shim.busy, ( + "an unrelated register_turn consumed caller A's reservation; A is now " + "evictable while it is still in setup" + ) + assert list(shim._reservations) == [reservation_a] + + # And a caller that *did* reserve consumes exactly its own reservation. + lease_c = shim.reserve() + assert len(shim._reservations) == 2 + token_c = lease_c.register_turn([], {}) + assert list(shim._reservations) == [reservation_a], ( + "registering through a lease must consume that lease's reservation and " + f"leave A's alone (left: {list(shim._reservations)})" + ) + shim.unregister_turn(token_c) + assert shim.busy # A is still in setup + + +# ------------------------------------------------- the cache across two loops + + +def _isolated_shim_cache(): + """Swap the process-global shim cache for empty ones, restoring on exit.""" + return _SwappedShimCache() + + +class _SwappedShimCache: + def __enter__(self): + self._shims = proxy_module._SHIMS + self._retired = proxy_module._RETIRED + proxy_module._SHIMS = OrderedDict() + proxy_module._RETIRED = [] + return proxy_module._SHIMS + + def __exit__(self, *exc): + # Restored unconditionally and before any fixture teardown runs, so the + # autouse guard above still compares the real cache with itself. + proxy_module._SHIMS = self._shims + proxy_module._RETIRED = self._retired + return False + + +def test_two_threads_racing_get_shim_build_exactly_one_shim(monkeypatch) -> None: + """The cache must be atomic across event loops, not just across coroutines. + + ``get_shim``'s check-and-insert used to rely on "this block performs no + awaits" -- which makes it atomic only within one event loop. ``usable_on`` + exists precisely because invocations may run "under their own + ``asyncio.run``" on separate threads, and two of those both missed the cache + and both constructed a ``ResponsesShim``. The second ``_SHIMS[key] = shim`` + orphaned the first, which then bound a port in ``start()`` while being + reachable from nothing -- not ``_evict_idle_shims``, not ``shutdown_shims``, + not ``_close_shims_at_exit`` -- leaking the socket for the life of the + process. + + ``start`` is stubbed to bind nothing *and* to leave ``_loop`` unset, so + ``usable_on`` is true for both loops and the test isolates the insert race + from the (separate, intended) loop-affinity discard. + """ + constructed: list[object] = [] + real_init = proxy_module.ResponsesShim.__init__ + + def instrumented_init(self, *args, **kwargs): + real_init(self, *args, **kwargs) + constructed.append(self) + # Widen the window the old code raced in: the GIL is released here, so + # the other thread reliably reaches its own check-and-insert. + time.sleep(0.02) + + async def fake_start(self): + self.url = self.url or "http://127.0.0.1:65535" + return self.url + + monkeypatch.setattr(proxy_module.ResponsesShim, "__init__", instrumented_init) + monkeypatch.setattr(proxy_module.ResponsesShim, "start", fake_start) + + barrier = threading.Barrier(2) + leases: dict[int, object] = {} + errors: list[BaseException] = [] + + def worker(index: int) -> None: + try: + barrier.wait(timeout=5) + leases[index] = asyncio.run( + proxy_module.get_shim("https://backend.invalid/v1", "backend-key") + ) + except BaseException as e: # noqa: BLE001 - reported below + errors.append(e) + + with _isolated_shim_cache() as cache: + threads = [threading.Thread(target=worker, args=(i,)) for i in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + cached = list(cache.values()) + retired = list(proxy_module._RETIRED) + + assert errors == [], errors + assert len(constructed) == 1, ( + f"{len(constructed)} shims were built for one cache key: the loser is " + "an orphan that binds a port no teardown path can ever find" + ) + assert len(cached) == 1 + assert retired == [] + assert leases[0].shim is leases[1].shim is cached[0], ( + "both threads must share the one cached shim" + ) + # Nothing constructed may be unreachable from the cache -- that is the leak. + assert {id(shim) for shim in constructed} == {id(shim) for shim in cached} + + +# ---------------------------------------------- what one charged call may cost + + +@pytest.mark.asyncio +async def test_a_repaired_backend_call_is_charged_once(monkeypatch) -> None: + """``on_model_call`` counts model calls, not HTTP attempts -- on purpose. + + One charge can become several requests: ``litellm.aresponses`` is given + ``num_retries``, and ``_call_backend_tolerating_reasoning`` may re-issue the + request without Codex's replayed ``reasoning`` items. Both are re-attempts + of a call that produced no response, so neither is a second *model* call; + charging them would spend a budget the user is not billed for and would make + ``max_llm_calls`` bind at a different point than on the ``adk`` runtime, + which counts flow calls while litellm retries underneath it. + + (``num_retries`` itself is applied inside ``litellm.aresponses``, which the + stub replaces, so what is asserted here is that the shim asks for it and + that the retry it *does* own is not charged.) + """ + from litellm import exceptions as litellm_exceptions + + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + charges: list[int] = [] + token = shim.register_turn([], {}, on_model_call=lambda: charges.append(1)) + + attempts: list[dict] = [] + + async def refuses_reasoning_once(**kwargs): + attempts.append(kwargs) + if any(item.get("type") == "reasoning" for item in kwargs["input"]): + raise litellm_exceptions.BadRequestError( + message="input[1].reasoning is not supported for model", + model="doubao-seed-1-6", + llm_provider="openai", + ) + return _text_response("done") + + monkeypatch.setattr(proxy_module.litellm, "aresponses", refuses_reasoning_once) + + async with _client(shim) as client: + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json={ + "model": "doubao-seed-1-6", + "stream": False, + "input": [ + _message("go"), + {"type": "reasoning", "summary": [{"text": "thinking"}]}, + ], + }, + ) + + assert response.status_code == 200 + assert len(attempts) == 2, "the repair retry must actually have happened" + assert charges == [1], ( + f"one model call was charged {len(charges)} times; the budget counts " + "calls, not the attempts a single call may cost" + ) + assert attempts[0]["num_retries"] == proxy_module._shim_num_retries() diff --git a/tests/runtime/codex/test_codex_tracing.py b/tests/runtime/codex/test_codex_tracing.py new file mode 100644 index 000000000..1f89006f7 --- /dev/null +++ b/tests/runtime/codex/test_codex_tracing.py @@ -0,0 +1,138 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tracing regression: a Codex turn must produce an indexable ``call_llm`` span. + +``_InMemoryExporter`` indexes a session *only* from spans literally named +``call_llm`` carrying ``gen_ai.session.id`` +(``veadk/tracing/telemetry/exporters/inmemory_exporter.py:82``). ADK opens that +span inside ``base_llm_flow``, which the Codex runtime replaces wholesale -- so +before the runtime opened its own, every Codex trace dump was ``[]``, +``OpentelemetryTracer.dump()`` wrote an empty list, and +``base_evaluator.build_eval_set`` then raised ``ValueError: Unsupported file +format`` at ``base_evaluator.py:419``. Nothing in between reported a problem. +""" + +from __future__ import annotations + +import sys +import uuid +from pathlib import Path + +import pytest +from google.adk.runners import Runner +from google.adk.sessions.in_memory_session_service import InMemorySessionService +from google.genai import types + +# The differential harness owns the offline Codex doubles (a fake SDK that +# drives the real shim over ASGI, and the scripted backend). Import them by +# path so this file runs on its own as well as inside a full-tree collection. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "differential")) + +import fake_codex_sdk # noqa: E402 +from scripted_backend import Round, ScriptedBackend # noqa: E402 + + +# Deliberately no per-test global ``TracerProvider`` swap: ADK's and VeADK's +# module-level tracers are ``ProxyTracer`` objects that memoize the first real +# provider they resolve, so replacing (or shutting down) the provider mid-session +# makes every later span vanish -- indistinguishable from the bug under test. +# ``OpentelemetryTracer`` attaches its own in-memory exporter to whichever +# provider is already active, which is order-independent. + + +@pytest.mark.asyncio +async def test_codex_run_emits_call_llm_span_with_session_id(monkeypatch) -> None: + from veadk import Agent + from veadk.runtime import get_runtime + from veadk.tracing.telemetry.opentelemetry_tracer import OpentelemetryTracer + + fake_codex_sdk.install_openai_codex_stub() + get_runtime.cache_clear() + + from veadk.runtime.codex import runtime as runtime_module + from veadk.runtime.codex.proxy import ResponsesShim + + backend = ScriptedBackend([Round(text="Beijing is sunny.", usage=(11, 7))]) + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + shim.url = f"http://shim-{uuid.uuid4().hex[:12]}" + fake_codex_sdk.SHIM_REGISTRY[shim.url] = shim + + async def fake_get_shim(api_base, api_key): + return shim + + monkeypatch.setattr( + "veadk.runtime.codex.proxy.litellm.aresponses", backend.as_aresponses() + ) + monkeypatch.setattr(runtime_module, "get_shim", fake_get_shim) + monkeypatch.setattr(runtime_module, "AsyncCodex", fake_codex_sdk.ShimDrivingCodex) + + tracer = OpentelemetryTracer(exporters=[]) + agent = Agent( + name="traced_codex_agent", + description="A traced codex agent.", + instruction="Answer the user.", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + tracers=[tracer], + ) + + session_id = f"session-{uuid.uuid4().hex[:8]}" + session_service = InMemorySessionService() + await session_service.create_session( + app_name="tracing", user_id="user", session_id=session_id + ) + runner = Runner(app_name="tracing", agent=agent, session_service=session_service) + + events = [ + event + async for event in runner.run_async( + user_id="user", + session_id=session_id, + new_message=types.Content(role="user", parts=[types.Part(text="weather?")]), + ) + ] + try: + assert events, "the codex run produced no events at all" + + exporter = tracer._inmemory_exporter._exporter + call_llm_spans = [s for s in exporter._spans if s.name == "call_llm"] + assert call_llm_spans, ( + f"no call_llm span: {sorted({s.name for s in exporter._spans})}" + ) + attributes = dict(call_llm_spans[0].attributes or {}) + assert attributes.get("gen_ai.session.id") == session_id, attributes + + # The property the exporter's session index -- and therefore every + # trace dump and every evaluation built from one -- actually depends on. + assert exporter.get_finished_spans(session_id), ( + "get_finished_spans() is empty, so OpentelemetryTracer.dump() would " + "write [] and base_evaluator.build_eval_set would raise" + ) + + # The span must carry the turn's tokens, not merely exist: an + # untokened call_llm span still produces a useless trace dump. + usage = { + key: value + for key, value in attributes.items() + if key.startswith("gen_ai.usage.") + } + assert usage, f"call_llm span carries no token usage: {sorted(attributes)}" + assert usage.get("gen_ai.usage.input_tokens") == 11, usage + assert usage.get("gen_ai.usage.output_tokens") == 7, usage + finally: + fake_codex_sdk.SHIM_REGISTRY.clear() + get_runtime.cache_clear() diff --git a/tests/runtime/codex/test_codex_turn_contract.py b/tests/runtime/codex/test_codex_turn_contract.py new file mode 100644 index 000000000..44c529588 --- /dev/null +++ b/tests/runtime/codex/test_codex_turn_contract.py @@ -0,0 +1,784 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end contract tests for one Codex turn, driven through the real runtime. + +The shim-level file next door (``test_codex_shim_rounds.py``) can express what +one HTTP request does, but not what the *runtime* guarantees across a whole +invocation. Everything here therefore runs ``CodexRuntime.run_async`` against +the differential suite's offline doubles -- the fake Codex SDK drives the real +shim over ``httpx.ASGITransport``, and a scripted backend stands in for the +model -- so no assertion here can be satisfied by a hand-built request body. + +That matters most for :func:`test_agent_turn_still_gets_tools_and_replay`, the +paired positive for the compaction-isolation tests: whatever marks a request as +"the agent's own turn", this test gets it the way production does. +""" + +from __future__ import annotations + +import asyncio +import contextvars +import shutil +import sys +import uuid +from pathlib import Path +from typing import Any + +import pytest +from google.adk.agents.invocation_context import LlmCallsLimitExceededError +from google.adk.agents.run_config import RunConfig +from google.adk.runners import Runner +from google.adk.sessions.in_memory_session_service import InMemorySessionService +from google.genai import types + +# Same import-by-path arrangement as `test_codex_tracing.py`: the differential +# suite owns the offline Codex doubles, and this file must run both standalone +# and inside a full-tree collection. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "differential")) + +import fake_codex_sdk # noqa: E402 +from scripted_backend import Round, ScriptedBackend # noqa: E402 + + +def record_fact(fact: str) -> dict: + """Record a fact.""" + return {"stored": fact} + + +class _BrokenStreamCodex(fake_codex_sdk.ShimDrivingCodex): + """Drives the shim normally, then drops the stream on the way out. + + Models a connection reset (or any SDK-side failure) arriving *after* the + shim has already recorded a turn error. That ordering is the whole point: + the runtime must still surface the recorded error rather than the transport + exception that happened to arrive last. + """ + + async def thread_start(self, **kwargs: Any) -> Any: + return _BrokenThread(await super().thread_start(**kwargs)) + + +class _BrokenThread: + def __init__(self, inner: Any) -> None: + self._inner = inner + + async def turn(self, input_items: Any, **kwargs: Any) -> Any: + return _BrokenTurn(await self._inner.turn(input_items, **kwargs)) + + +class _BrokenTurn: + def __init__(self, inner: Any) -> None: + self._inner = inner + self.id = inner.id + + async def interrupt(self) -> None: + return None + + def stream(self) -> Any: + async def _gen(): + async for note in self._inner.stream(): + yield note + raise RuntimeError("codex stream dropped") + + return _gen() + + +async def _run_turn( + monkeypatch, + *, + plan, + agent_kwargs: dict | None = None, + run_config: RunConfig | None = None, + codex_class: type | None = None, +): + """Run one Codex invocation offline; return (events, session, backend, error).""" + from veadk import Agent + from veadk.runtime import get_runtime + + # Must precede the `runtime` import: that module imports `openai_codex` at + # module scope, and the stub is what stands in for it when it is absent. + fake_codex_sdk.install_openai_codex_stub() + get_runtime.cache_clear() + + from veadk.runtime.codex import runtime as runtime_module + from veadk.runtime.codex.proxy import ResponsesShim + + backend = ScriptedBackend(plan, arm="codex") + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + shim.url = f"http://shim-{uuid.uuid4().hex[:12]}" + fake_codex_sdk.SHIM_REGISTRY[shim.url] = shim + + async def fake_get_shim(api_base, api_key): + return shim + + monkeypatch.setattr( + "veadk.runtime.codex.proxy.litellm.aresponses", backend.as_aresponses() + ) + monkeypatch.setattr(runtime_module, "get_shim", fake_get_shim) + monkeypatch.setattr( + runtime_module, "AsyncCodex", codex_class or fake_codex_sdk.ShimDrivingCodex + ) + + agent = Agent( + name="contract_agent", + description="A codex contract agent.", + instruction="Answer the user.", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + **(agent_kwargs or {}), + ) + + session_id = f"session-{uuid.uuid4().hex[:8]}" + session_service = InMemorySessionService() + await session_service.create_session( + app_name="contract", user_id="user", session_id=session_id + ) + runner = Runner(app_name="contract", agent=agent, session_service=session_service) + + events: list[Any] = [] + error: BaseException | None = None + try: + try: + async for event in runner.run_async( + user_id="user", + session_id=session_id, + new_message=types.Content(role="user", parts=[types.Part(text="go")]), + **({"run_config": run_config} if run_config is not None else {}), + ): + events.append(event) + except BaseException as e: # noqa: BLE001 - the error IS the observable + error = e + + session = await session_service.get_session( + app_name="contract", user_id="user", session_id=session_id + ) + finally: + # Process-global, and this file runs under `pytest -n 16`: a leaked + # registry entry or a memoized runtime would follow every later test in + # this worker. + fake_codex_sdk.SHIM_REGISTRY.clear() + get_runtime.cache_clear() + return events, session, backend, error + + +# ------------------------------------------ paired positive for compaction gating + + +@pytest.mark.asyncio +async def test_agent_turn_still_gets_tools_and_replay(monkeypatch) -> None: + """The agent's own turn must keep getting ADK tools and transcript replay. + + Paired positive for + ``test_codex_shim_rounds.py::test_compaction_request_gets_no_adk_tools_*``: + those assert the shim leaves a *non*-agent-turn request alone, and without + this one they could all be satisfied by never injecting anything at all, + which would silently disable ADK tools for the codex runtime entirely. + + Deliberately end-to-end. A hand-built POST could be made to satisfy any + gating rule by construction; only a request the runtime itself produced + proves the real agent turn is still recognized as one. + """ + events, _session, backend, error = await _run_turn( + monkeypatch, + plan=( + Round(tool_calls=(("record_fact", {"fact": "sky is blue"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), + ), + agent_kwargs={"tools": [record_fact]}, + ) + assert error is None, error + assert events, "the codex turn produced no events" + + assert backend.calls, "the scripted backend was never called" + assert "record_fact" in backend.calls[0].tool_names, ( + "the agent's ADK tool was not advertised on its own turn -- gating " + f"over-corrected into never injecting: {backend.calls[0].tool_names}" + ) + + assert len(backend.calls) >= 2, ( + f"expected a tool round then an answer round: {backend.calls}" + ) + assert backend.calls[1].tool_records == ( + ("function_call", "record_fact"), + ("function_response", "record_fact"), + ), ( + "the agent turn's own tool transcript was not replayed to the model, so " + f"it would re-issue the call: {backend.calls[1].tool_records}" + ) + + +@pytest.mark.asyncio +async def test_agent_turn_replays_across_two_codex_requests(monkeypatch) -> None: + """The cross-request half of the same guarantee, end to end. + + The test above stays inside one Codex request (the shim's own tool loop). + This one forces Codex to issue a *second* request under the same turn token, + by having the model ask for a tool the shim has no executor for: the shim + hands that call back to Codex, which answers it and re-POSTs. Only + ``turn_context.state.replay_items`` can put the earlier ADK pair into that + second request, since Codex rebuilds ``input`` from its own thread and never + saw it. + + Without this, gating tool-transcript replay on "is this the agent's turn?" + could pass every compaction-isolation test while quietly disabling replay + for the shape that motivated it. + """ + events, _session, backend, error = await _run_turn( + monkeypatch, + plan=( + # Round 1: an ADK tool the shim executes itself. + Round(tool_calls=(("record_fact", {"fact": "blue"}),), usage=(0, 0)), + # Round 2: a call the shim cannot execute -> returned to Codex, + # which answers it locally and re-POSTs. Second request, one token. + Round(tool_calls=(("codex_owned_tool", {"q": "x"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), + ), + agent_kwargs={"tools": [record_fact]}, + ) + assert error is None, error + assert events, "the codex turn produced no events" + assert len(backend.calls) >= 3, ( + f"codex never issued a second request: {len(backend.calls)} backend calls" + ) + + replayed = backend.calls[2].tool_records + assert ("function_call", "record_fact") in replayed, ( + "the first request's ADK tool pair was not replayed into Codex's second " + f"request, so the model would re-issue the call: {replayed}" + ) + assert ("function_response", "record_fact") in replayed, replayed + + +# ---------------------------------------- the recorded turn error must survive + + +@pytest.mark.asyncio +async def test_budget_error_survives_a_failing_turn(monkeypatch) -> None: + """``max_llm_calls`` must fire even when the turn also fails. + + The shim serves backend calls on the server's task, so an exhausted budget + cannot propagate from there: it is recorded on the turn state and returned + to Codex as a 429, and ``run_async`` re-reads it afterwards. But that read + happens on only one of three exits -- if the stream then raises, the + ``finally``'s ``unregister_turn`` drops the recorded error and the caller + sees the transport exception instead. ``max_llm_calls`` then silently does + nothing on every failure path, and ``on_model_error`` callbacks are handed + the wrong exception. + """ + _events, _session, _backend, error = await _run_turn( + monkeypatch, + plan=( + Round(tool_calls=(("record_fact", {"fact": "blue"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), + ), + agent_kwargs={"tools": [record_fact]}, + run_config=RunConfig(max_llm_calls=1), + codex_class=_BrokenStreamCodex, + ) + + assert isinstance(error, LlmCallsLimitExceededError), ( + "the recorded max_llm_calls error was dropped when the turn failed; " + f"the caller saw {type(error).__name__}: {error!r}" + ) + + +# ------------------------------------------------- a tool-only turn still counts + + +@pytest.mark.asyncio +async def test_tool_only_turn_propagates_state_delta_and_usage(monkeypatch) -> None: + """A turn that ends without assistant text must not lose its bookkeeping. + + The runtime builds exactly one merged response per turn -- that is where + after-model callbacks run, where the turn's ``usage_metadata`` is attached, + and where a callback's ``callback_context.state`` writes become a + ``state_delta``. Dropping that event because it carries no text throws all + of it away: ``output_key``-style state writes never reach the session and + the turn's tokens never reach usage accounting. Marking the event partial + instead would not help, since partial events are never persisted. + """ + + def _before_model(callback_context, llm_request): # noqa: ANN001 + callback_context.state["seen_by_callback"] = "yes" + return None + + events, session, _backend, error = await _run_turn( + monkeypatch, + plan=( + Round(tool_calls=(("record_fact", {"fact": "blue"}),), usage=(11, 5)), + # The turn ends on a tool round with no assistant text at all. + Round(usage=(7, 3)), + ), + agent_kwargs={ + "tools": [record_fact], + "before_model_callback": _before_model, + }, + ) + assert error is None, error + + state = dict(getattr(session, "state", None) or {}) + assert state.get("seen_by_callback") == "yes", ( + "a tool-only turn dropped the merged event, so the callback's state " + f"write never reached the session: {state}" + ) + + total = sum( + int(getattr(getattr(e, "usage_metadata", None), "total_token_count", 0) or 0) + for e in events + ) + assert total == 26, ( + f"a tool-only turn reported {total} tokens; the turn spent 11+5 and 7+3" + ) + + +# ----------------------------------- the workspace an ADK tool is able to find + + +class _Rendezvous: + """Releases its callers only once ``parties`` of them have arrived. + + Turns "two turns ran" into "two turns were *inside their tool* at the same + instant", which is the only arrangement in which a shared-state mechanism + can be caught handing one tenant another's workspace. + """ + + def __init__(self, parties: int) -> None: + self._parties = parties + self._arrived = 0 + self._all_here = asyncio.Event() + + async def wait(self, timeout: float = 60.0) -> None: + self._arrived += 1 + if self._arrived >= self._parties: + self._all_here.set() + await asyncio.wait_for(self._all_here.wait(), timeout) + + +#: A workspace no turn owns, bound in the context the detached driver below +#: hands to the shim. It stands in for the first-ever invocation's value, which +#: is what a production shim's server task carries forever. +_DECOY_WORKSPACE = "/tmp/veadk-decoy-workspace-owned-by-no-turn" + + +class _DetachedShimCodex(fake_codex_sdk.ShimDrivingCodex): + """Drives the shim from a task that does not descend from the invocation. + + ``httpx.ASGITransport`` calls the shim's handler inline, on the caller's + task, which makes the offline harness *friendlier than production*: there + the handler runs on a task descended from the uvicorn server task, which + ``asyncio.create_task`` created -- and whose context it snapshotted -- when + the first invocation in the process started the shim. A ContextVar the + invocation merely sets is therefore not visible to a tool; the first + invocation's value is. (Measured: with the var set in ``run_async``, three + later turns' tools all read the first turn's workspace. It is the same + asymmetry that forces the shim to capture an OTel context in + ``register_turn`` and re-attach it around tool execution.) + + So the request is issued here from a task created in a context where + :data:`_DECOY_WORKSPACE` is bound. Any design that lets the tool read the + workspace out of ambient task context now reports the decoy instead of the + turn's own directory, in the test as it would in production. + """ + + async def thread_start(self, **kwargs: Any) -> Any: + return _DetachedThread(await super().thread_start(**kwargs)) + + +class _DetachedThread: + def __init__(self, inner: Any) -> None: + self._inner = inner + + async def turn(self, input_items: Any, **kwargs: Any) -> Any: + return _DetachedTurn(await self._inner.turn(input_items, **kwargs)) + + +class _DetachedTurn: + def __init__(self, inner: Any) -> None: + self._inner = inner + self.id = inner.id + + async def interrupt(self) -> None: + await self._inner.interrupt() + + def stream(self) -> Any: + from veadk.runtime.codex.workspace import bind_workspace + + inner = self._inner.stream() + queue: asyncio.Queue = asyncio.Queue() + done = object() + + async def _pump() -> None: + try: + async for note in inner: + await queue.put(note) + except BaseException as e: # noqa: BLE001 - relayed to the consumer + await queue.put(e) + finally: + await queue.put(done) + + # A fresh copy per turn: a Context cannot be entered twice at once, and + # these two turns overlap. + with bind_workspace(_DECOY_WORKSPACE): + detached = contextvars.copy_context() + task = detached.run(asyncio.create_task, _pump()) + + async def _gen(): + try: + while True: + item = await queue.get() + if item is done: + return + if isinstance(item, BaseException): + raise item + yield item + finally: + if not task.done(): + task.cancel() + + return _gen() + + +#: Set by the concurrency test; ``None`` leaves ``stage_dataset`` sequential. +_RENDEZVOUS: _Rendezvous | None = None + +#: What each call of ``stage_dataset`` saw, in completion order. +_STAGED: list[dict[str, str]] = [] + + +async def stage_dataset(label: str) -> dict: + """Write this tenant's dataset into your working directory. + + Args: + label (str): The tenant this turn belongs to. + + Returns: + dict: A receipt with the workspace-relative path. + """ + from veadk.runtime.codex import current_workspace + + if _RENDEZVOUS is not None: + # Both tools are now in flight; whatever each reads next, it reads + # while the other turn's tool is also inside its executor. + await _RENDEZVOUS.wait() + workspace = current_workspace() + # Recorded before the write, so a wrong-but-plausible path is reported as + # the wrong path rather than as an unwritable one. + _STAGED.append({"label": label, "workspace": workspace or ""}) + if workspace is None: + return {"status": "error", "message": "no codex workspace on this call"} + path = Path(workspace) / "staged.txt" + try: + path.write_text(label, encoding="utf-8") + except OSError as e: + return {"status": "error", "message": str(e)} + return {"status": "ok", "path": "staged.txt"} + + +async def _run_two_turns_concurrently(monkeypatch, labels: tuple[str, str]): + """Drive two invocations at once, through one shim, and return their events. + + Deliberately *one* shim for both turns (that is what a server does: the + shim is memoized per backend), no ``workspace_root``, and no + ``reuse_workspace`` -- so each turn gets its own session-keyed workspace, + which is the arrangement the examples now rely on. + """ + from veadk import Agent + from veadk.runtime import get_runtime + + fake_codex_sdk.install_openai_codex_stub() + get_runtime.cache_clear() + + from veadk.runtime.codex import runtime as runtime_module + from veadk.runtime.codex.proxy import ResponsesShim + + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + shim.url = f"http://shim-{uuid.uuid4().hex[:12]}" + fake_codex_sdk.SHIM_REGISTRY[shim.url] = shim + + async def fake_get_shim(api_base, api_key): + return shim + + # One scripted backend per turn, since the plan cursor is per-arm and the + # two turns interleave. Routed by the tenant name in each agent's own + # instruction, which reaches the wire as the request's `instructions` on + # every request of that turn. (Not the prompt text: the offline fake reads + # `TextInput.value`, an attribute only its stub has, so prompts arrive + # empty when the real openai-codex SDK is installed.) + backends = { + label: ScriptedBackend( + ( + Round(tool_calls=(("stage_dataset", {"label": label}),), usage=(0, 0)), + Round(text=f"staged for {label}", usage=(3, 2)), + ), + arm="codex", + ) + for label in labels + } + adapters = {label: backend.as_aresponses() for label, backend in backends.items()} + + async def dispatch(**kwargs: Any) -> Any: + instructions = str(kwargs.get("instructions") or "") + for label, adapter in adapters.items(): + if f"tenant {label}" in instructions: + return await adapter(**kwargs) + raise AssertionError(f"no tenant in request instructions: {instructions!r}") + + monkeypatch.setattr("veadk.runtime.codex.proxy.litellm.aresponses", dispatch) + monkeypatch.setattr(runtime_module, "get_shim", fake_get_shim) + # Not `ShimDrivingCodex`: the shim must be driven from a task that does not + # descend from either invocation, the way uvicorn drives it in production. + monkeypatch.setattr(runtime_module, "AsyncCodex", _DetachedShimCodex) + + session_service = InMemorySessionService() + + async def _drive(label: str) -> list[Any]: + agent = Agent( + name="tenant_agent", + description="A codex contract agent.", + instruction=f"Answer the user. You serve tenant {label}.", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + tools=[stage_dataset], + ) + session_id = f"session-{label}-{uuid.uuid4().hex[:8]}" + await session_service.create_session( + app_name="contract", user_id=label, session_id=session_id + ) + runner = Runner( + app_name="contract", agent=agent, session_service=session_service + ) + return [ + event + async for event in runner.run_async( + user_id=label, + session_id=session_id, + new_message=types.Content( + role="user", parts=[types.Part(text=f"stage data for {label}")] + ), + ) + ] + + try: + return await asyncio.gather(*(_drive(label) for label in labels)) + finally: + fake_codex_sdk.SHIM_REGISTRY.clear() + get_runtime.cache_clear() + + +@pytest.mark.asyncio +async def test_concurrent_turns_each_see_their_own_workspace(monkeypatch) -> None: + """Two turns at once, each tool writing into its own session's workspace. + + This is the multi-tenant requirement in one test. Before + ``current_workspace()`` existed, the examples had to pin ``workspace_root`` + *and* ``reuse_workspace=True`` purely so their ADK tools could find the + directory Codex was working in -- which put every session in one shared + directory. Nothing here is pinned: both turns take the default, per-session + workspace, and the tool still finds the right one. + + The rendezvous is what makes it a concurrency test rather than two + sequential ones: neither tool reads the workspace until both are inside + their executor, so a mechanism that keeps "the current workspace" anywhere + process-wide (or that lets the shim's own task context supply it) hands at + least one of them the wrong path. + """ + global _RENDEZVOUS + _STAGED.clear() + _RENDEZVOUS = _Rendezvous(2) + try: + results = await _run_two_turns_concurrently(monkeypatch, ("alpha", "beta")) + finally: + _RENDEZVOUS = None + + assert all(events for events in results), "a turn produced no events" + assert len(_STAGED) == 2, ( + "both tools should have run and recorded a workspace (a missing entry " + f"means one turn never reached its tool): {_STAGED}" + ) + + seen = {entry["label"]: entry["workspace"] for entry in _STAGED} + assert seen.keys() == {"alpha", "beta"} + assert all(seen.values()), ( + "a tool could not find the turn's workspace, so an ADK tool has no " + f"supported way to hand Codex a file: {_STAGED}" + ) + assert seen["alpha"] != seen["beta"], ( + "both tenants were handed the same workspace -- per-session isolation " + f"is gone and one tenant can read the other's files: {seen}" + ) + for label, workspace in seen.items(): + staged = Path(workspace) / "staged.txt" + assert staged.read_text(encoding="utf-8") == label, ( + f"{label}'s file landed in the wrong workspace: " + f"{staged} holds {staged.read_text(encoding='utf-8')!r}" + ) + + # Nothing was pinned, so both workspaces must be the runtime's own + # per-session directories under the process-owned root. + from veadk.runtime.codex import runtime as runtime_module + + root = Path(runtime_module._SESSION_WORKSPACE_ROOT) + for workspace in seen.values(): + assert Path(workspace).parent == root, ( + f"{workspace} is not a per-session workspace under {root}; the " + "turn fell back to a shared directory" + ) + shutil.rmtree(workspace, ignore_errors=True) + + +@pytest.mark.asyncio +async def test_tool_workspace_is_bound_per_call_not_inherited(monkeypatch) -> None: + """The binding must survive the shim's task, and never leak between turns. + + The offline arrangement above drives the shim over ``ASGITransport``, so + its handler runs on the invocation's own task -- friendlier than production, + where the handler descends from the uvicorn server task whose context was + snapshotted when the *first* invocation in the process started the shim. + Measured against a real shim, a ContextVar merely set in ``run_async`` made + every later turn's tool read the first turn's workspace: a silent + cross-tenant leak, not a miss. + + This reproduces that topology directly. The wrapped executor is invoked + from a task created while a *different* workspace is bound, exactly as the + shim's handler would be, and must still report its own turn's. + """ + from veadk.runtime.codex.workspace import ( + bind_workspace, + bind_workspace_to_executors, + current_workspace, + ) + + async def _probe(args: dict[str, Any], call_id: str) -> str: + return str(current_workspace()) + + wrapped = bind_workspace_to_executors({"probe": _probe}, "/tmp/tenant-b")["probe"] + + with bind_workspace("/tmp/tenant-a"): + # The task inherits tenant A's context, the way the shim's server task + # inherited the first invocation's. + task = asyncio.create_task(wrapped({}, "call-1")) + assert await task == "/tmp/tenant-b", ( + "the executor read the workspace from the calling task's context " + "instead of its own turn's" + ) + + assert current_workspace() is None, "the binding outlived its call" + + +def test_current_workspace_is_none_outside_a_codex_turn() -> None: + """Outside a turn the accessor reports absence rather than raising. + + An ADK tool is not codex-specific: the same object is run by the default + LLM flow, by ``AgentTool`` and by unit tests, so raising here would make a + workspace-aware tool unusable everywhere else. ``None`` is one branch, and + the tool can return a model-readable error of its own. + """ + from veadk.runtime.codex import current_workspace + + assert current_workspace() is None + + +# --------------------------------- what the model is told about its own tools + + +@pytest.mark.asyncio +async def test_turn_corrects_codex_prompt_about_unusable_tools(monkeypatch) -> None: + """The developer channel must correct Codex's prompt on two tools. + + Codex keeps its own ~21KB system prompt (the runtime never sends + ``base_instructions``, which would *replace* it), and that prompt tells the + model to edit files with ``apply_patch`` -- a tool the shim never forwards, + because it forwards only ``function``-typed tools. It also leaves + ``request_user_input`` advertised, though an ADK invocation has nobody to + answer it, so a call to it ends the turn having done nothing. + + Both example agents had to counter-instruct this in their own prompts. The + note belongs to the runtime, on the additive developer channel, next to the + agent's instruction. + """ + _events, _session, backend, error = await _run_turn( + monkeypatch, + plan=(Round(text="Done.", usage=(4, 2)),), + ) + assert error is None, error + assert backend.calls, "the scripted backend was never called" + + instructions = backend.calls[0].system_instruction + assert "apply_patch" in instructions, ( + "nothing tells the model apply_patch is unavailable, so it will call a " + f"tool the backend was never given: {instructions!r}" + ) + assert "exec_command" in instructions, instructions + assert "request_user_input" in instructions, ( + "nothing tells the model its question cannot be answered mid-turn: " + f"{instructions!r}" + ) + # The note is additive, never a replacement: the agent's own instruction + # and identity must still be there. + assert "Answer the user." in instructions, instructions + assert "contract_agent" in instructions, instructions + + +# ------------------------------------------ importing must not touch the disk + + +def test_importing_the_runtime_creates_no_workspace_root(tmp_path) -> None: + """Importing the module must not create a temp directory. + + ``_SESSION_WORKSPACE_ROOT`` used to be a module-level + ``tempfile.mkdtemp(...)``, so every process that merely imported this + module made a ``veadk-codex-workspaces-*`` root in ``$TMPDIR`` -- reclaimed + by an ``atexit`` hook, which a ``SIGKILL`` (an OOM kill, a torn-down xdist + worker) never runs. A smoke run found several orphaned roots predating it, + and ``pytest -n 16`` leaves one per killed worker. + + Run in a subprocess with a private ``TMPDIR``: the root is process-global + and this suite has already served turns, so nothing in-process can still + observe a first import -- and a shared ``$TMPDIR`` would see roots made by + any other process on the machine. + """ + import os + import subprocess + + differential = str(Path(__file__).resolve().parents[1] / "differential") + program = ( + "import sys\n" + f"sys.path.insert(0, {differential!r})\n" + "import fake_codex_sdk\n" + "fake_codex_sdk.install_openai_codex_stub()\n" + "import veadk.runtime.codex.runtime as runtime\n" + "print(runtime._session_workspace_root)\n" + ) + private_tmp = tmp_path / "tmp" + private_tmp.mkdir() + result = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=180, + env={**os.environ, "TMPDIR": str(private_tmp)}, + ) + assert result.returncode == 0, result.stderr[-2000:] + assert result.stdout.strip() == "None", ( + f"the module built a workspace root on import: {result.stdout.strip()}" + ) + assert not list(private_tmp.iterdir()), ( + "importing the module created " + f"{[p.name for p in private_tmp.iterdir()]} in $TMPDIR" + ) diff --git a/tests/runtime/differential/conftest.py b/tests/runtime/differential/conftest.py new file mode 100644 index 000000000..ad86b877c --- /dev/null +++ b/tests/runtime/differential/conftest.py @@ -0,0 +1,460 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Differential ("对拍") harness: run one agent config through both runtimes. + +The suite's whole value rests on the comparison actually being able to fail, so +three mechanisms guard it: + +1. :func:`normalize_events` is a **closed allowlist**. An event it cannot + classify raises instead of being dropped, so a new Codex event type forces a + human decision rather than silently widening the excluded set. +2. Every exclusion carries a paired positive assertion somewhere in the suite + (see ``test_runtime_parity.py``'s "paired positive" tests). +3. ``test_parity_harness.py`` injects faults into the Codex arm and asserts the + comparison raises for each. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, Callable, Iterable, Sequence + +import pytest +from google.genai import types + +import fake_codex_sdk +import scripted_backend +from scripted_backend import RecordedCall, Round, ScriptedBackend + +#: Event classifications deliberately kept out of the equivalence class. Each +#: one has a paired positive assertion in ``test_runtime_parity.py``. +EXCLUDED_KINDS = frozenset({"thought", "delta", "codex_lifecycle", "adk_state"}) + +#: The comparable event vocabulary. +COMPARED_KINDS = frozenset({"text", "function_call", "function_response"}) + +_CODEX_LIFECYCLE_TYPES = frozenset( + { + "item_started", + "item_completed", + "message_delta", + "command_output", + "file_change_output", + "mcp_progress", + "plan_delta", + "reasoning_delta", + "file_change_patch", + "plan_item", + "plan_update", + "turn_started", + "turn_complete", + "token_usage", + "approval_review", + "context_compacted", + "model_rerouted", + "error", + } +) + + +@dataclass +class RunOutcome: + """Everything one arm produced, normalized for comparison.""" + + arm: str + final_text: str = "" + tool_calls: tuple[tuple[str, str], ...] = () + tool_responses: tuple[str, ...] = () + state_delta: dict[str, Any] = field(default_factory=dict) + session_state: dict[str, Any] = field(default_factory=dict) + usage: tuple[int, int, int] = (0, 0, 0) + event_kinds: frozenset[str] = frozenset() + span_names: frozenset[str] | None = None + calls: list[RecordedCall] = field(default_factory=list) + events: list[Any] = field(default_factory=list) + authors: tuple[str, ...] = () + declared_usage: tuple[int, int] = (0, 0) + error: BaseException | None = None + + +def classify_event(event: Any) -> str: + """Classify one ADK event. Raises on anything the suite has not seen. + + This is the closed allowlist. Widening it is a deliberate act. + """ + if getattr(event, "partial", False): + return "delta" + + content = getattr(event, "content", None) + parts = list(getattr(content, "parts", None) or []) if content else [] + parts = [p for p in parts if p is not None] + if parts: + if any( + getattr(p, "text", None) is not None and getattr(p, "thought", False) + for p in parts + ): + return "thought" + if any(getattr(p, "function_call", None) is not None for p in parts): + return "function_call" + if any(getattr(p, "function_response", None) is not None for p in parts): + return "function_response" + if any(getattr(p, "text", None) is not None for p in parts): + return "text" + + if getattr(event, "error_code", None): + return "error" + + metadata = getattr(event, "custom_metadata", None) or {} + codex_type = metadata.get("codex_event_type") + if codex_type is not None: + if codex_type not in _CODEX_LIFECYCLE_TYPES: + raise AssertionError( + f"unclassified codex lifecycle event type {codex_type!r}: {event!r}" + ) + return "codex_lifecycle" + + actions = getattr(event, "actions", None) + if actions is not None and content is None: + return "adk_state" + + raise AssertionError(f"unclassified event: {event!r}") + + +def normalize_events(events: Sequence[Any], agent_name: str) -> dict[str, Any]: + """Reduce a raw event stream to the comparable observables.""" + kinds: set[str] = set() + texts: list[str] = [] + tool_calls: list[tuple[str, str]] = [] + tool_responses: list[str] = [] + state_delta: dict[str, Any] = {} + prompt = candidates = total = 0 + + for event in events: + kind = classify_event(event) + if kind in COMPARED_KINDS: + kinds.add(kind) + + usage = getattr(event, "usage_metadata", None) + if usage is not None: + prompt += int(getattr(usage, "prompt_token_count", 0) or 0) + candidates += int(getattr(usage, "candidates_token_count", 0) or 0) + total += int(getattr(usage, "total_token_count", 0) or 0) + + actions = getattr(event, "actions", None) + if actions is not None: + state_delta.update(dict(getattr(actions, "state_delta", None) or {})) + + content = getattr(event, "content", None) + parts = [p for p in (getattr(content, "parts", None) or []) if p is not None] + if kind == "text": + texts.extend( + str(p.text) + for p in parts + if getattr(p, "text", None) and not getattr(p, "thought", False) + ) + elif kind == "function_call": + for part in parts: + call = getattr(part, "function_call", None) + if call is not None and call.name: + # Compare by value, never by author: codex's item_to_events + # uses role "user" for tool responses. + tool_calls.append((str(call.name), _stable(call.args or {}))) + elif kind == "function_response": + for part in parts: + response = getattr(part, "function_response", None) + if response is not None and response.name: + tool_responses.append(str(response.name)) + + return { + "kinds": frozenset(kinds), + "final_text": "".join(texts).strip(), + "tool_calls": tuple(tool_calls), + "tool_responses": tuple(tool_responses), + "state_delta": state_delta, + "usage": (prompt, candidates, total), + } + + +def _stable(value: Any) -> str: + import json + + try: + return json.dumps(value, sort_keys=True, default=str) + except Exception: # noqa: BLE001 + return str(value) + + +# --------------------------------------------------------------- comparison + + +def compare_runs( + adk: RunOutcome, + codex: RunOutcome, + *, + expected_usage: tuple[int, int] | None = None, +) -> None: + """Assert the two arms are in the same equivalence class. + + Every mismatch is collected before raising, so one systemic divergence + (e.g. missing spans) cannot mask the others. + """ + problems: list[str] = [] + + def check(label: str, left: Any, right: Any) -> None: + if left != right: + problems.append(f"{label}: adk={left!r} codex={right!r}") + + if type(adk.error) is not type(codex.error): + problems.append( + f"error type: adk={type(adk.error).__name__} " + f"codex={type(codex.error).__name__}" + ) + if adk.error is not None or codex.error is not None: + if problems: + raise AssertionError("runtime parity mismatch:\n " + "\n ".join(problems)) + return + + check("final_text", adk.final_text, codex.final_text) + check("tool_calls", adk.tool_calls, codex.tool_calls) + check("tool_responses", adk.tool_responses, codex.tool_responses) + check("state_delta", adk.state_delta, codex.state_delta) + check("session_state(output_key)", adk.session_state, codex.session_state) + + if not adk.event_kinds <= codex.event_kinds: + problems.append( + f"event kinds: adk {sorted(adk.event_kinds)} not a subset of " + f"codex {sorted(codex.event_kinds)}" + ) + + if expected_usage is not None: + want = (expected_usage[0], expected_usage[1], sum(expected_usage)) + if adk.usage != want: + problems.append(f"adk usage: got {adk.usage}, plan declares {want}") + if codex.usage != want: + problems.append(f"codex usage: got {codex.usage}, plan declares {want}") + + if adk.span_names is not None and codex.span_names is not None: + if not adk.span_names <= codex.span_names: + problems.append( + f"span names: adk {sorted(adk.span_names)} not a subset of " + f"codex {sorted(codex.span_names)}" + ) + for arm, names in (("adk", adk.span_names), ("codex", codex.span_names)): + if "call_llm" not in names: + problems.append(f"{arm} spans missing call_llm: {sorted(names)}") + + for index, (left, right) in enumerate(zip(adk.calls, codex.calls)): + if left.comparable() != right.comparable(): + for key, value in left.comparable().items(): + other = right.comparable()[key] + if value != other: + problems.append( + f"request[{index}].{key}: adk={value!r} codex={other!r}" + ) + if len(adk.calls) != len(codex.calls): + problems.append(f"request count: adk={len(adk.calls)} codex={len(codex.calls)}") + + if problems: + raise AssertionError("runtime parity mismatch:\n " + "\n ".join(problems)) + + +# ------------------------------------------------------------------ running + + +class ParityRunner: + """Runs one agent configuration through one runtime, offline.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.monkeypatch = monkeypatch + + async def run( + self, + arm: str, + *, + plan: Iterable[Round], + agent_kwargs: Callable[[ScriptedBackend], dict] | dict | None = None, + run_config: Any = None, + capture_spans: bool = False, + codex_fault: Callable[[RunOutcome], None] | None = None, + user_text: str = "do the thing", + session_id: str | None = None, + ) -> RunOutcome: + from google.adk.runners import Runner + from google.adk.sessions.in_memory_session_service import InMemorySessionService + + from veadk import Agent + from veadk.runtime import get_runtime + + get_runtime.cache_clear() + backend = ScriptedBackend(plan, arm=arm) + kwargs = dict( + agent_kwargs(backend) if callable(agent_kwargs) else (agent_kwargs or {}) + ) + agent_name = kwargs.pop("name", "parity_agent") + session_id = session_id or f"session-{uuid.uuid4().hex[:8]}" + + tracer = None + spans_before = 0 + if capture_spans: + from veadk.tracing.telemetry.opentelemetry_tracer import OpentelemetryTracer + + # Attaches to the active provider (creating one only if none + # exists), so the memoized proxy tracers keep resolving correctly. + tracer = OpentelemetryTracer(exporters=[]) + spans_before = len(tracer._inmemory_exporter._exporter._spans) + kwargs.setdefault("tracers", [tracer]) + + if arm == "codex": + self._install_codex_doubles(backend) + + # The ADK arm consumes the plan through a BaseLlm; the Codex arm + # resolves the model from model_name and reaches the plan through the + # shim. Passing Agent(model=...) under codex is a hard error in + # veadk.runtime.compat, so the two arms differ only here. + if arm == "adk": + kwargs["model"] = backend.as_base_llm() + agent = Agent( + name=agent_name, + description="A differential parity agent.", + instruction="Answer the user.", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime=arm, + **kwargs, + ) + + session_service = InMemorySessionService() + await session_service.create_session( + app_name="parity", user_id="user", session_id=session_id + ) + runner = Runner(app_name="parity", agent=agent, session_service=session_service) + + outcome = RunOutcome(arm=arm, calls=backend.calls) + events: list[Any] = [] + try: + async for event in runner.run_async( + user_id="user", + session_id=session_id, + new_message=types.Content( + role="user", parts=[types.Part(text=user_text)] + ), + **({"run_config": run_config} if run_config is not None else {}), + ): + events.append(event) + except BaseException as e: # noqa: BLE001 - the error IS an observable + outcome.error = e + + session = await session_service.get_session( + app_name="parity", user_id="user", session_id=session_id + ) + outcome.events = events + outcome.authors = tuple(getattr(e, "author", "") for e in events) + normalized = normalize_events(events, agent_name) + outcome.final_text = normalized["final_text"] + outcome.tool_calls = normalized["tool_calls"] + outcome.tool_responses = normalized["tool_responses"] + outcome.state_delta = normalized["state_delta"] + outcome.event_kinds = normalized["kinds"] + outcome.usage = normalized["usage"] + outcome.session_state = dict(getattr(session, "state", None) or {}) + outcome.declared_usage = backend.declared_usage_total + if tracer is not None: + spans = tracer._inmemory_exporter._exporter._spans[spans_before:] + outcome.span_names = frozenset(span.name for span in spans) + + if arm == "codex" and codex_fault is not None: + codex_fault(outcome) + return outcome + + def _install_codex_doubles(self, backend: ScriptedBackend) -> None: + """Wire the Codex arm to the in-process shim with no socket or binary.""" + fake_codex_sdk.install_openai_codex_stub() + + from veadk.runtime.codex import runtime as runtime_module + from veadk.runtime.codex.proxy import ResponsesShim + + shim = ResponsesShim("https://backend.invalid/v1", "backend-key") + shim.url = f"http://shim-{uuid.uuid4().hex[:12]}" + fake_codex_sdk.SHIM_REGISTRY[shim.url] = shim + self.monkeypatch.setattr( + "veadk.runtime.codex.proxy.litellm.aresponses", backend.as_aresponses() + ) + + async def fake_get_shim(api_base: str, api_key: str) -> Any: + return shim + + self.monkeypatch.setattr(runtime_module, "get_shim", fake_get_shim) + self.monkeypatch.setattr( + runtime_module, "AsyncCodex", fake_codex_sdk.ShimDrivingCodex + ) + self.shim = shim + + +@pytest.fixture +def parity_runner(monkeypatch: pytest.MonkeyPatch) -> Any: + """Function-scoped runner: no shared state can leak between rows.""" + fake_codex_sdk.REQUEST_LOG.clear() + # The compat layer dedupes warnings per (agent id, field) process-wide, so + # a row asserting on a warning must start from a clean slate. + from veadk.runtime.compat import reset_warning_state + + reset_warning_state() + runner = ParityRunner(monkeypatch) + yield runner + reset_warning_state() + fake_codex_sdk.SHIM_REGISTRY.clear() + fake_codex_sdk.REQUEST_LOG.clear() + from veadk.runtime import get_runtime + + get_runtime.cache_clear() + + +@pytest.fixture +def compare() -> Any: + """Expose :func:`compare_runs` so tests never import conftest directly.""" + return compare_runs + + +@pytest.fixture +def event_classifier() -> Any: + return classify_event + + +# NOTE: this suite deliberately does *not* swap OpenTelemetry's process-global +# ``TracerProvider`` per test. ADK's and VeADK's module-level tracers are +# ``ProxyTracer`` objects that memoize the first real provider they resolve, so +# installing a second provider mid-session silently sends every later span into +# a detached (often already shut down) pipeline -- which looks exactly like "the +# runtime emits no spans". Each ``capture_spans`` row instead attaches its own +# ``InMemoryExporter`` to whichever provider is already active and reads back +# only the spans produced during its own run. + + +__all__ = [ + "COMPARED_KINDS", + "EXCLUDED_KINDS", + "ParityRunner", + "RecordedCall", + "Round", + "RunOutcome", + "ScriptedBackend", + "classify_event", + "compare_runs", + "fake_codex_sdk", + "normalize_events", + "scripted_backend", +] diff --git a/tests/runtime/differential/fake_codex_sdk.py b/tests/runtime/differential/fake_codex_sdk.py new file mode 100644 index 000000000..03c44dfff --- /dev/null +++ b/tests/runtime/differential/fake_codex_sdk.py @@ -0,0 +1,611 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Codex SDK double that actually drives the Responses shim over HTTP. + +Every previous Codex test cut OpenClaw at one of three mock boundaries: +``litellm.aresponses`` (never exercising the shim), a fake ``AsyncCodex`` +(never exercising the shim *or* the wire), or ``httpx.ASGITransport`` against +the shim alone (never exercising the runtime). Each boundary hid a different +class of bug, and the union of the three hid the interesting ones entirely. + +:class:`ShimDrivingCodex` collapses all three. It replaces ``AsyncCodex`` in +:mod:`veadk.runtime.codex.runtime` and then behaves like the real Codex CLI: + +* it reads its bearer token from ``config.env["VEADK_CODEX_API_KEY"]`` and its + endpoint from the ``config.toml`` that ``_prepare_codex_home`` generated under + ``config.env["CODEX_HOME"]`` -- so config generation is under test rather than + stubbed out; +* it POSTs a real ``/v1/responses`` request with ``stream: True`` through + ``httpx.ASGITransport`` (in-process, no socket, xdist-safe), which means + ``proxy._synth_sse`` runs on every differential test for free; +* it parses the synthesized SSE stream back into items; +* it implements the minimal Codex agentic loop: a ``function_call`` item it has + no executor for is answered locally and the turn is re-POSTed with the call + and its output appended -- a second request under one token; +* it emits real ``openai_codex`` notification models when the SDK is importable + and name-compatible shims when it is not. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import tomllib +import types as pytypes +from typing import Any, AsyncIterator + +import httpx + +#: ``{shim_url: ResponsesShim}``. The runtime writes the shim URL into +#: ``config.toml``; the fake reads it back and needs the ASGI app behind it. +SHIM_REGISTRY: dict[str, Any] = {} + +#: Recorded POST bodies, in order, for tests that assert on the wire shape. +REQUEST_LOG: list[dict[str, Any]] = [] + + +def openai_codex_available() -> bool: + """Whether the real ``openai-codex`` distribution is importable.""" + if "openai_codex" in sys.modules: + return not getattr(sys.modules["openai_codex"], "__veadk_stub__", False) + try: + return importlib.util.find_spec("openai_codex") is not None + except (ImportError, ValueError): + return False + + +def install_openai_codex_stub() -> bool: + """Register a minimal ``openai_codex`` stub when the real SDK is absent. + + ``veadk.runtime.codex.runtime`` imports the SDK at module scope, so without + this the whole differential suite would silently skip on any machine that + did not ``uv sync --all-extras``. The stub only has to satisfy the names the + runtime imports; ``AsyncCodex`` itself is always replaced by + :class:`ShimDrivingCodex`. + + Call this from a *fixture*, never at module import time: pytest finishes + collecting (and therefore evaluating every ``importorskip("openai_codex")``) + before the first test runs, so installing it here cannot turn a legitimate + skip into a spurious pass. + + Returns: + bool: ``True`` if a stub is now in ``sys.modules``. + """ + if openai_codex_available(): + return False + if isinstance(sys.modules.get("openai_codex"), pytypes.ModuleType) and getattr( + sys.modules["openai_codex"], "__veadk_stub__", False + ): + return True + + from enum import Enum + + class _StrEnum(str, Enum): + pass + + class ApprovalMode(_StrEnum): + deny_all = "deny_all" + auto_review = "auto_review" + + class Sandbox(_StrEnum): + read_only = "read_only" + workspace_write = "workspace_write" + full_access = "full_access" + + class Personality(_StrEnum): + none = "none" + friendly = "friendly" + pragmatic = "pragmatic" + + class ReasoningEffort(_StrEnum): + minimal = "minimal" + low = "low" + medium = "medium" + high = "high" + xhigh = "xhigh" + + class CodexConfig: + def __init__(self, *, cwd: str | None = None, env: dict | None = None) -> None: + self.cwd = cwd + self.env = dict(env or {}) + + class _Input: + def __init__(self, value: Any, name: Any = None) -> None: + if name is None: + self.value = value + else: + self.name, self.value = value, name + + class TextInput(_Input): + pass + + class ImageInput(_Input): + pass + + class LocalImageInput(_Input): + pass + + class MentionInput(_Input): + pass + + module = pytypes.ModuleType("openai_codex") + module.__veadk_stub__ = True # type: ignore[attr-defined] + generated = pytypes.ModuleType("openai_codex.generated") + generated.__veadk_stub__ = True # type: ignore[attr-defined] + v2_all = pytypes.ModuleType("openai_codex.generated.v2_all") + v2_all.__veadk_stub__ = True # type: ignore[attr-defined] + + for name, value in ( + ("ApprovalMode", ApprovalMode), + ("Sandbox", Sandbox), + ("CodexConfig", CodexConfig), + ("TextInput", TextInput), + ("ImageInput", ImageInput), + ("LocalImageInput", LocalImageInput), + ("MentionInput", MentionInput), + ("AsyncCodex", ShimDrivingCodex), + ): + setattr(module, name, value) + for name, value in ( + ("Personality", Personality), + ("ReasoningEffort", ReasoningEffort), + ): + setattr(v2_all, name, value) + for name in _NOTIFICATION_NAMES: + setattr(v2_all, name, _shim_notification_class(name)) + module.generated = generated # type: ignore[attr-defined] + generated.v2_all = v2_all # type: ignore[attr-defined] + + sys.modules["openai_codex"] = module + sys.modules["openai_codex.generated"] = generated + sys.modules["openai_codex.generated.v2_all"] = v2_all + return True + + +_NOTIFICATION_NAMES = ( + "TurnStartedNotification", + "TurnCompletedNotification", + "ItemStartedNotification", + "ItemCompletedNotification", + "AgentMessageDeltaNotification", + "ReasoningSummaryTextDeltaNotification", + "ThreadTokenUsageUpdatedNotification", + "ErrorNotification", +) + +_SHIM_CLASSES: dict[str, type] = {} + + +def _shim_notification_class(name: str) -> type: + """A name-compatible stand-in; ``translate`` dispatches on the class name.""" + existing = _SHIM_CLASSES.get(name) + if existing is not None: + return existing + + def __init__(self: Any, payload: dict[str, Any]) -> None: + self._payload = dict(payload) + for key, value in payload.items(): + setattr(self, key, value) + + def model_dump(self: Any, *args: Any, **kwargs: Any) -> dict[str, Any]: + return dict(self._payload) + + def __repr__(self: Any) -> str: + return f"{name}({self._payload!r})" + + cls = type( + name, (), {"__init__": __init__, "model_dump": model_dump, "__repr__": __repr__} + ) + _SHIM_CLASSES[name] = cls + return cls + + +def make_notification(name: str, payload: dict[str, Any]) -> Any: + """Build ``name`` from the real SDK when possible, else a shim. + + Real construction is attempted through ``model_validate`` so this stays + schema-agnostic: if the SDK's model rejects the payload we fall back rather + than fail, and :mod:`tests.runtime.codex.test_codex_sdk_protocol` is the + place that asserts real construction actually works. + """ + if openai_codex_available(): + try: + from openai_codex.generated import v2_all # type: ignore + + model = getattr(v2_all, name, None) + if model is not None and hasattr(model, "model_validate"): + return model.model_validate(payload) + except Exception: # noqa: BLE001 - a schema drift must not break tests + pass + return _shim_notification_class(name)(payload) + + +class _Note: + """The ``note`` wrapper the SDK stream yields; the runtime reads ``payload``.""" + + def __init__(self, payload: Any) -> None: + self.payload = payload + + +class ShimDrivingCodex: + """``AsyncCodex`` replacement that speaks HTTP to the in-process shim.""" + + #: Set by tests to make the fake ask for a tool the shim cannot execute, + #: forcing the two-requests-under-one-token path. + max_agent_loops = 4 + + def __init__(self, *, config: Any) -> None: + self.config = config + + async def __aenter__(self) -> "ShimDrivingCodex": + return self + + async def __aexit__(self, *exc: Any) -> None: + return None + + async def thread_start(self, **kwargs: Any) -> "_Thread": + return _Thread(self.config, kwargs) + + +class _Thread: + def __init__(self, config: Any, start_kwargs: dict[str, Any]) -> None: + self.config = config + self.start_kwargs = start_kwargs + + async def turn(self, input_items: Any, **kwargs: Any) -> "_Turn": + return _Turn(self.config, self.start_kwargs, input_items, kwargs) + + +class _Turn: + id = "turn-1" + + def __init__( + self, + config: Any, + start_kwargs: dict[str, Any], + input_items: Any, + turn_kwargs: dict[str, Any], + ) -> None: + self.config = config + self.start_kwargs = start_kwargs + self.input_items = input_items + self.turn_kwargs = turn_kwargs + + def stream(self) -> AsyncIterator[_Note]: + return _drive(self) + + async def interrupt(self) -> None: + return None + + +def _prompt_text(input_items: Any) -> str: + texts: list[str] = [] + for item in input_items or []: + value = getattr(item, "value", None) + if isinstance(value, str) and type(item).__name__ == "TextInput": + texts.append(value) + return "\n".join(texts) + + +def shim_endpoint_from_codex_home(codex_home: str) -> str: + """Read the provider ``base_url`` back out of the generated ``config.toml``. + + Doing this (rather than being handed the URL) is what puts + ``runtime._prepare_codex_home`` under test. + """ + with open(os.path.join(codex_home, "config.toml"), "rb") as handle: + config = tomllib.load(handle) + return str(config["model_providers"]["veadk"]["base_url"]) + + +async def _drive(turn: _Turn) -> AsyncIterator[_Note]: + env = dict(getattr(turn.config, "env", None) or {}) + token = env["VEADK_CODEX_API_KEY"] + base_url = shim_endpoint_from_codex_home(env["CODEX_HOME"]) + shim_url = base_url[: -len("/v1")] if base_url.endswith("/v1") else base_url + shim = SHIM_REGISTRY.get(shim_url) + if shim is None: + raise AssertionError( + f"no registered shim for {shim_url!r} (from config.toml {base_url!r}); " + f"known: {sorted(SHIM_REGISTRY)}" + ) + + instructions = "\n\n".join( + part + for part in ( + turn.start_kwargs.get("base_instructions"), + turn.start_kwargs.get("developer_instructions"), + ) + if part + ) + conversation: list[dict[str, Any]] = [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": _prompt_text(turn.input_items)}], + } + ] + + yield _Note( + make_notification( + "TurnStartedNotification", + {"turn": {"id": turn.id, "status": "in_progress"}}, + ) + ) + + transport = httpx.ASGITransport(app=shim._app) + # `ThreadTokenUsage.last` is the model call that just finished; `total` is + # cumulative for the thread. Keeping them distinct is what lets the suite + # tell a per-call design apart from a cumulative one -- a fake that sets + # both to the same block cannot detect double counting in either direction. + running: dict[str, int] = {} + try: + async with httpx.AsyncClient( + transport=transport, base_url=shim_url, timeout=30.0 + ) as client: + for _ in range(ShimDrivingCodex.max_agent_loops): + body = { + "model": str(turn.start_kwargs.get("model") or "scripted-model"), + "stream": True, + "instructions": instructions, + "input": conversation, + "tools": [], + "store": False, + } + REQUEST_LOG.append(json.loads(json.dumps(body))) + response = await client.post( + "/v1/responses", + headers={"Authorization": f"Bearer {token}"}, + json=body, + ) + if response.status_code != 200: + yield _Note( + make_notification( + "ErrorNotification", + { + "error": { + "code": str(response.status_code), + "message": response.text, + }, + "will_retry": False, + }, + ) + ) + break + + items, completed = _parse_sse(response.text) + last = _usage_block(dict((completed or {}).get("usage") or {})) + if any(last.values()): + for key, value in last.items(): + running[key] = running.get(key, 0) + value + yield _Note( + make_notification( + "ThreadTokenUsageUpdatedNotification", + { + "turn_id": turn.id, + "model_context_window": 128000, + "token_usage": { + "last": last, + "total": dict(running), + }, + }, + ) + ) + + pending: list[dict[str, Any]] = [] + for item in items: + if item.get("type") == "function_call": + pending.append(item) + for note in _item_notifications(turn.id, item): + yield note + + if not pending: + break + + # Minimal Codex agentic loop: answer the call locally and + # re-POST with the pair appended -- a second request under the + # same turn token, which is the shape that breaks tool history. + for call in pending: + call_id = call.get("call_id") or call.get("id") + conversation.append( + { + "type": "function_call", + "call_id": call_id, + "id": call.get("id") or call_id, + "name": call.get("name"), + "arguments": call.get("arguments") or "{}", + "status": "completed", + } + ) + conversation.append( + { + "type": "function_call_output", + "call_id": call_id, + "output": json.dumps( + {"status": "completed", "output": "codex-executed"} + ), + } + ) + finally: + pass + + yield _Note( + make_notification( + "TurnCompletedNotification", + {"turn": {"id": turn.id, "status": "completed", "error": None}}, + ) + ) + + +def _usage_block(usage: dict[str, Any]) -> dict[str, int]: + input_tokens = int(usage.get("input_tokens") or 0) + output_tokens = int(usage.get("output_tokens") or 0) + return { + "input_tokens": input_tokens, + "cached_input_tokens": int(usage.get("cached_input_tokens") or 0), + "output_tokens": output_tokens, + "reasoning_output_tokens": int(usage.get("reasoning_output_tokens") or 0), + "total_tokens": int(usage.get("total_tokens") or input_tokens + output_tokens), + } + + +def _item_notifications(turn_id: str, item: dict[str, Any]) -> list[_Note]: + """Map one Responses output item onto the Codex thread-item lifecycle.""" + item_id = str(item.get("id") or "item") + itype = item.get("type") + + if itype == "message": + text = "\n".join( + str(part.get("text") or "") + for part in item.get("content") or [] + if isinstance(part, dict) + ) + thread_item = {"id": item_id, "type": "agentMessage", "text": text} + return [ + _Note( + make_notification( + "ItemStartedNotification", + { + "turn_id": turn_id, + "item": {"id": item_id, "type": "agentMessage", "text": ""}, + }, + ) + ), + _Note( + make_notification( + "AgentMessageDeltaNotification", + {"turn_id": turn_id, "item_id": item_id, "delta": text}, + ) + ), + _Note( + make_notification( + "ItemCompletedNotification", + {"turn_id": turn_id, "item": thread_item}, + ) + ), + ] + + if itype == "reasoning": + summary = [ + {"text": str(entry.get("text") or "")} + for entry in item.get("summary") or [] + if isinstance(entry, dict) + ] + thread_item = {"id": item_id, "type": "reasoning", "summary": summary} + notes = [ + _Note( + make_notification( + "ItemStartedNotification", + { + "turn_id": turn_id, + "item": {"id": item_id, "type": "reasoning", "summary": []}, + }, + ) + ) + ] + for entry in summary: + notes.append( + _Note( + make_notification( + "ReasoningSummaryTextDeltaNotification", + { + "turn_id": turn_id, + "item_id": item_id, + "delta": entry["text"], + }, + ) + ) + ) + notes.append( + _Note( + make_notification( + "ItemCompletedNotification", + {"turn_id": turn_id, "item": thread_item}, + ) + ) + ) + return notes + + if itype == "function_call": + thread_item = { + "id": item_id, + "type": "dynamicToolCall", + "namespace": "codex", + "tool": str(item.get("name") or "tool"), + "arguments": item.get("arguments") or "{}", + "content_items": [{"text": "codex-executed"}], + "success": True, + "status": "completed", + } + return [ + _Note( + make_notification( + "ItemStartedNotification", + {"turn_id": turn_id, "item": {**thread_item, "status": None}}, + ) + ), + _Note( + make_notification( + "ItemCompletedNotification", + {"turn_id": turn_id, "item": thread_item}, + ) + ), + ] + + return [] + + +def _parse_sse(text: str) -> tuple[list[dict[str, Any]], dict[str, Any] | None]: + """Parse a ``text/event-stream`` body into (done items, completed response). + + Items are taken from ``response.output_item.done`` -- i.e. what Codex would + actually act on -- not from the terminal payload, so a tool call dropped by + the synthesizer is invisible to the fake exactly as it would be to Codex. + """ + items: list[dict[str, Any]] = [] + completed: dict[str, Any] | None = None + for frame in text.split("\n\n"): + payload = None + for line in frame.splitlines(): + if line.startswith("data:"): + payload = json.loads(line[len("data:") :].strip()) + if not isinstance(payload, dict): + continue + if payload.get("type") == "response.output_item.done": + item = payload.get("item") + if isinstance(item, dict): + items.append(item) + elif payload.get("type") == "response.completed": + completed = payload.get("response") or {} + return items, completed + + +def parse_sse_events(text: str) -> list[dict[str, Any]]: + """Every SSE frame as ``{"event": name, "data": {...}}``, in wire order.""" + events: list[dict[str, Any]] = [] + for frame in text.split("\n\n"): + if not frame.strip(): + continue + name = None + data = None + for line in frame.splitlines(): + if line.startswith("event:"): + name = line[len("event:") :].strip() + elif line.startswith("data:"): + data = json.loads(line[len("data:") :].strip()) + events.append({"event": name, "data": data}) + return events diff --git a/tests/runtime/differential/scripted_backend.py b/tests/runtime/differential/scripted_backend.py new file mode 100644 index 000000000..5c44a2692 --- /dev/null +++ b/tests/runtime/differential/scripted_backend.py @@ -0,0 +1,450 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A declarative model backend shared by the ADK and Codex differential arms. + +The two runtimes consume the model at different protocol levels: + +- ``runtime="adk"`` calls ``BaseLlm.generate_content_async(llm_request)`` and + consumes ``LlmResponse`` objects; +- ``runtime="codex"`` reaches the model through the Responses shim, which calls + ``litellm.aresponses(**kwargs)`` and consumes a Responses ``dict``. + +There is therefore no single object both arms can share. What *is* shareable is +a declarative :class:`Round` plan plus two adapters that replay it, and -- more +importantly -- a normalized record of **what each arm asked the model for**. + +That record (:class:`RecordedCall`) is the single decision that makes the +differential suite able to catch silent no-ops. With a scripted model, dropping +``temperature`` (or the tool history, or OpenClaw instruction) does not change +the produced text at all, so a pure output comparison is blind to it. Recording +the inputs makes those rows testable. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, AsyncGenerator, Callable, Iterable + +from google.genai import types + +_HISTORY_OPEN = "" +_HISTORY_CLOSE = "" +_CURRENT_OPEN = "" +_CURRENT_CLOSE = "" + +#: Text the backend replies with once the scripted plan is exhausted. A +#: terminal text round (rather than an error) keeps a runaway agentic loop +#: bounded without masking the assertion that actually failed. +EXHAUSTED_TEXT = "[scripted-backend-exhausted]" + + +@dataclass(frozen=True) +class Round: + """One scripted model reply. + + Attributes: + text: Assistant text for this round, or ``None`` for a tool-only round. + tool_calls: ``(tool_name, args)`` pairs the model asks for, in order. + usage: ``(input_tokens, output_tokens)`` reported for this round. + raises: When set, the backend raises this instead of replying. + """ + + text: str | None = None + texts: tuple[str, ...] = () + tool_calls: tuple[tuple[str, dict[str, Any]], ...] = () + usage: tuple[int, int] = (0, 0) + raises: BaseException | None = None + + @property + def reply_texts(self) -> tuple[str, ...]: + """All assistant text chunks this round emits, in order.""" + if self.texts: + return self.texts + return () if self.text is None else (self.text,) + + +@dataclass(frozen=True) +class RecordedCall: + """Normalized view of one model request, comparable across both arms. + + ``system_instruction`` is deliberately *excluded* from :meth:`comparable`: + the two runtimes legitimately wrap it in different boilerplate. Rows that + care about it assert containment of a specific fragment instead. + """ + + arm: str + tool_names: tuple[str, ...] + history_texts: tuple[str, ...] + current_text: str + tool_records: tuple[tuple[str, str], ...] + temperature: float | None + max_output_tokens: int | None + stop_sequences: tuple[str, ...] + system_instruction: str + + def comparable(self) -> dict[str, Any]: + """The subset of fields that must be identical in both arms.""" + return { + "tool_names": self.tool_names, + "history_texts": self.history_texts, + "current_text": self.current_text, + "tool_records": self.tool_records, + "temperature": self.temperature, + "max_output_tokens": self.max_output_tokens, + "stop_sequences": self.stop_sequences, + } + + +class ScriptedBackend: + """Replays a :class:`Round` plan into either runtime, recording its inputs. + + Instantiate one per arm (the round cursor and the call log are per-arm), + from the same shared plan. + """ + + def __init__(self, rounds: Iterable[Round], *, arm: str = "unknown") -> None: + self.rounds: list[Round] = list(rounds) + self.arm = arm + self.calls: list[RecordedCall] = [] + self._cursor = 0 + + # ---------------------------------------------------------------- plan + + @property + def declared_usage_total(self) -> tuple[int, int]: + """``(prompt_tokens, output_tokens)`` the plan declares in total. + + Only rounds actually consumed count, so a plan whose tail is never + reached (an early-terminating arm) is not silently credited. + """ + used = self.rounds[: self._cursor] + return (sum(r.usage[0] for r in used), sum(r.usage[1] for r in used)) + + def _next(self) -> Round: + if self._cursor >= len(self.rounds): + self._cursor += 1 + return Round(text=EXHAUSTED_TEXT) + rnd = self.rounds[self._cursor] + self._cursor += 1 + return rnd + + # ------------------------------------------------------------- adk arm + + def as_base_llm(self, model: str = "scripted-model") -> Any: + """A ``BaseLlm`` that replays the plan for ``runtime="adk"``.""" + from google.adk.models.base_llm import BaseLlm + from google.adk.models.llm_response import LlmResponse + + backend = self + + class _ScriptedLlm(BaseLlm): + async def generate_content_async( # type: ignore[override] + self, llm_request: Any, stream: bool = False + ) -> AsyncGenerator[Any, None]: + index = backend._cursor + backend.calls.append(backend._record_adk(llm_request)) + rnd = backend._next() + if rnd.raises is not None: + raise rnd.raises + parts: list[types.Part] = [] + for offset, (name, args) in enumerate(rnd.tool_calls): + parts.append( + types.Part( + function_call=types.FunctionCall( + id=f"call-{index}-{offset}", name=name, args=dict(args) + ) + ) + ) + for chunk in rnd.reply_texts: + parts.append(types.Part(text=chunk)) + yield LlmResponse( + content=types.Content(role="model", parts=parts), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=rnd.usage[0], + candidates_token_count=rnd.usage[1], + total_token_count=rnd.usage[0] + rnd.usage[1], + ), + ) + + return _ScriptedLlm(model=model) + + # ----------------------------------------------------------- codex arm + + def as_aresponses(self) -> Callable[..., Any]: + """A ``litellm.aresponses`` replacement replaying the plan.""" + + async def aresponses(**kwargs: Any) -> dict[str, Any]: + index = self._cursor + self.calls.append(self._record_codex(kwargs)) + rnd = self._next() + if rnd.raises is not None: + raise rnd.raises + output: list[dict[str, Any]] = [] + for offset, (name, args) in enumerate(rnd.tool_calls): + output.append( + { + "id": f"fc-{index}-{offset}", + "call_id": f"call-{index}-{offset}", + "type": "function_call", + "name": name, + "arguments": json.dumps(dict(args)), + "status": "completed", + } + ) + for chunk_index, chunk in enumerate(rnd.reply_texts): + output.append( + { + "id": f"msg-{index}-{chunk_index}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": chunk, + "annotations": [], + } + ], + } + ) + return { + "id": f"resp-{index}", + "object": "response", + "model": str(kwargs.get("model") or "scripted-model"), + "status": "completed", + "output": output, + "usage": { + "input_tokens": rnd.usage[0], + "cached_input_tokens": 0, + "output_tokens": rnd.usage[1], + "reasoning_output_tokens": 0, + "total_tokens": rnd.usage[0] + rnd.usage[1], + }, + } + + return aresponses + + # ------------------------------------------------------- normalization + + def _record_adk(self, llm_request: Any) -> RecordedCall: + config = getattr(llm_request, "config", None) + tool_names: list[str] = [] + for tool in getattr(config, "tools", None) or []: + for declaration in getattr(tool, "function_declarations", None) or []: + if declaration.name: + tool_names.append(str(declaration.name)) + + contents = list(getattr(llm_request, "contents", None) or []) + history, current, tool_records = _split_contents(contents) + return RecordedCall( + arm="adk", + tool_names=tuple(tool_names), + history_texts=tuple(history), + current_text=current, + tool_records=tuple(tool_records), + temperature=getattr(config, "temperature", None), + max_output_tokens=getattr(config, "max_output_tokens", None), + stop_sequences=tuple(getattr(config, "stop_sequences", None) or ()), + system_instruction=_system_instruction_text( + getattr(config, "system_instruction", None) + ), + ) + + def _record_codex(self, kwargs: dict[str, Any]) -> RecordedCall: + tool_names = tuple( + str(tool.get("name")) + for tool in kwargs.get("tools") or [] + if isinstance(tool, dict) and tool.get("type") == "function" + ) + + history: list[str] = [] + current = "" + tool_records: list[tuple[str, str]] = [] + call_names: dict[str, str] = {} + + for item in kwargs.get("input") or []: + if isinstance(item, str): + item_history, item_current, item_tools = _split_prompt(item) + history.extend(item_history) + current = item_current or current + tool_records.extend(item_tools) + continue + if not isinstance(item, dict): + continue + itype = item.get("type") + if itype == "function_call": + name = str(item.get("name") or "") + call_names[str(item.get("call_id") or item.get("id") or "")] = name + tool_records.append(("function_call", name)) + elif itype == "function_call_output": + key = str(item.get("call_id") or item.get("id") or "") + tool_records.append(("function_response", call_names.get(key, ""))) + elif itype in (None, "message"): + text = _codex_message_text(item) + item_history, item_current, item_tools = _split_prompt(text) + if item.get("role") == "assistant": + if text: + history.append(text) + continue + history.extend(item_history) + current = item_current or current + tool_records.extend(item_tools) + + return RecordedCall( + arm="codex", + tool_names=tool_names, + history_texts=tuple(history), + current_text=current, + tool_records=tuple(tool_records), + temperature=kwargs.get("temperature"), + max_output_tokens=kwargs.get("max_output_tokens"), + stop_sequences=tuple(kwargs.get("stop_sequences") or ()), + system_instruction=str(kwargs.get("instructions") or ""), + ) + + +# --------------------------------------------------------------- helpers + + +def _system_instruction_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + parts = getattr(value, "parts", None) + if parts is not None: + return "\n".join(p.text for p in parts if getattr(p, "text", None)).strip() + if isinstance(value, (list, tuple)): + return "\n".join(_system_instruction_text(v) for v in value).strip() + return str(value) + + +def _content_texts(content: Any) -> list[str]: + texts: list[str] = [] + for part in getattr(content, "parts", None) or []: + text = getattr(part, "text", None) + if text and not getattr(part, "thought", False): + texts.append(str(text)) + return texts + + +def _split_contents( + contents: list[Any], +) -> tuple[list[str], str, list[tuple[str, str]]]: + """Split ADK ``contents`` into (history texts, current text, tool records). + + ``current_text`` is the *last* user text turn -- the message that triggered + this invocation. Everything else is history. This mirrors how the Codex arm + serializes the same conversation, so the two are comparable. + """ + tool_records: list[tuple[str, str]] = [] + text_turns: list[tuple[int, str, str]] = [] # (index, role, text) + for index, content in enumerate(contents): + role = str(getattr(content, "role", "") or "") + for part in getattr(content, "parts", None) or []: + call = getattr(part, "function_call", None) + if call is not None and getattr(call, "name", None): + tool_records.append(("function_call", str(call.name))) + response = getattr(part, "function_response", None) + if response is not None and getattr(response, "name", None): + tool_records.append(("function_response", str(response.name))) + for text in _content_texts(content): + text_turns.append((index, role, text)) + + current = "" + current_at = -1 + for position in range(len(text_turns) - 1, -1, -1): + if text_turns[position][1] == "user": + current = text_turns[position][2] + current_at = position + break + history = [t for i, (_, _, t) in enumerate(text_turns) if i != current_at] + return history, current, tool_records + + +def _codex_message_text(item: dict[str, Any]) -> str: + content = item.get("content") + if isinstance(content, str): + return content + texts: list[str] = [] + for part in content or []: + if isinstance(part, str): + texts.append(part) + elif isinstance(part, dict) and part.get("text"): + texts.append(str(part["text"])) + return "\n".join(texts) + + +def _split_prompt(text: str) -> tuple[list[str], str, list[tuple[str, str]]]: + """Recover (history texts, current text, tool records) from a Codex prompt. + + ``veadk.runtime.codex.translate.build_prompt_from_llm_request`` serializes + the whole ADK conversation into one prompt string wrapped in + ```` / ```` JSON blocks. Parsing it + back is what lets a Codex request be compared with an ADK one. + """ + if not text: + return [], "", [] + history_json = _between(text, _HISTORY_OPEN, _HISTORY_CLOSE) + current_json = _between(text, _CURRENT_OPEN, _CURRENT_CLOSE) + if history_json is None and current_json is None: + return [], text.strip(), [] + + history: list[str] = [] + tool_records: list[tuple[str, str]] = [] + for record in _loads(history_json) or []: + texts, records = _parse_prompt_parts(record.get("parts") or []) + history.extend(texts) + tool_records.extend(records) + current_texts, current_records = _parse_prompt_parts(_loads(current_json) or []) + tool_records.extend(current_records) + return history, "\n".join(current_texts).strip(), tool_records + + +def _parse_prompt_parts( + parts: Any, +) -> tuple[list[str], list[tuple[str, str]]]: + texts: list[str] = [] + records: list[tuple[str, str]] = [] + for part in parts or []: + if not isinstance(part, dict): + continue + if part.get("type") == "text" and part.get("text"): + texts.append(str(part["text"])) + elif part.get("type") == "function_call": + records.append(("function_call", str(part.get("name") or ""))) + elif part.get("type") == "function_response": + records.append(("function_response", str(part.get("name") or ""))) + return texts, records + + +def _between(text: str, open_tag: str, close_tag: str) -> str | None: + start = text.find(open_tag) + if start < 0: + return None + end = text.find(close_tag, start) + if end < 0: + return None + return text[start + len(open_tag) : end] + + +def _loads(raw: str | None) -> Any: + if not raw: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None diff --git a/tests/runtime/differential/test_parity_harness.py b/tests/runtime/differential/test_parity_harness.py new file mode 100644 index 000000000..3bae1b450 --- /dev/null +++ b/tests/runtime/differential/test_parity_harness.py @@ -0,0 +1,351 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Meta-tests: prove the differential harness can fail. + +A comparison harness that cannot detect a divergence turns every row in +``test_runtime_parity.py`` permanently green, which is strictly worse than +having no suite at all. These tests inject known faults into the Codex arm and +assert the comparison raises for each of them. + +They are deliberately the first thing in the suite: written before the matrix, +and the first thing to check when a parity row unexpectedly goes green. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from google.adk.events.event import Event +from google.genai import types + +import conftest as harness # noqa: F401 (documented below) +from scripted_backend import RecordedCall, Round, ScriptedBackend + +# The tool round declares zero tokens on purpose. The Codex arm reports only +# the *final* backend response's usage (the shim's internal tool loop does not +# accumulate across rounds -- see the `usage_accounting` row in +# ``test_runtime_parity.py``), and these meta-tests must fail only because of +# the fault they inject, never because of an unrelated known divergence. +_PLAN = [ + Round(tool_calls=(("record_fact", {"fact": "sky is blue"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), +] + + +def record_fact(fact: str) -> dict: + """Record a fact.""" + return {"stored": fact} + + +def _agent_kwargs(_backend: ScriptedBackend) -> dict: + return {"tools": [record_fact], "output_key": "answer"} + + +async def _both(parity_runner, **kwargs): + adk = await parity_runner.run("adk", plan=_PLAN, agent_kwargs=_agent_kwargs) + codex = await parity_runner.run( + "codex", plan=_PLAN, agent_kwargs=_agent_kwargs, **kwargs + ) + return adk, codex + + +# ------------------------------------------------------------- fault matrix + + +def _drop_tool_call(outcome) -> None: + outcome.tool_calls = () + outcome.tool_responses = () + + +def _drop_state_delta(outcome) -> None: + outcome.state_delta = {} + outcome.session_state = {} + + +def _different_text(outcome) -> None: + outcome.final_text = "something else entirely" + + +def _zero_usage(outcome) -> None: + outcome.usage = (0, 0, 0) + + +def _drop_request(outcome) -> None: + outcome.calls = outcome.calls[:1] + + +def _drop_temperature(outcome) -> None: + outcome.calls = [ + RecordedCall(**{**call.__dict__, "temperature": None}) for call in outcome.calls + ] + + +def _drop_history(outcome) -> None: + outcome.calls = [ + RecordedCall(**{**call.__dict__, "history_texts": ()}) for call in outcome.calls + ] + + +def _drop_tool_history(outcome) -> None: + outcome.calls = [ + RecordedCall(**{**call.__dict__, "tool_records": ()}) for call in outcome.calls + ] + + +def _drop_event_kind(outcome) -> None: + outcome.event_kinds = frozenset() + + +def _drop_spans(outcome) -> None: + outcome.span_names = frozenset({"invocation"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fault", + [ + pytest.param(_drop_tool_call, id="drops_tool_call"), + pytest.param(_drop_state_delta, id="drops_state_delta"), + pytest.param(_different_text, id="different_text"), + pytest.param(_zero_usage, id="zero_usage"), + pytest.param(_drop_request, id="drops_a_request"), + pytest.param(_drop_temperature, id="drops_temperature"), + pytest.param(_drop_history, id="drops_history"), + pytest.param(_drop_tool_history, id="drops_tool_history"), + pytest.param(_drop_event_kind, id="drops_event_kind"), + ], +) +async def test_harness_detects_injected_divergence( + parity_runner, compare, fault +) -> None: + """Each injected fault must make the comparison raise.""" + adk, codex = await _both(parity_runner, codex_fault=fault) + assert adk.error is None, adk.error + assert codex.error is None, codex.error + + # A fault on temperature/history only shows up if the un-faulted values + # differ from the ADK arm's, so seed the ADK side with a real value first. + if fault is _drop_temperature: + adk.calls = [ + RecordedCall(**{**call.__dict__, "temperature": 0.1}) for call in adk.calls + ] + if fault is _drop_history: + adk.calls = [ + RecordedCall(**{**call.__dict__, "history_texts": ("earlier",)}) + for call in adk.calls + ] + if fault is _drop_tool_history: + seeded = (("function_call", "record_fact"),) + adk.calls = [ + RecordedCall(**{**call.__dict__, "tool_records": seeded}) + for call in adk.calls + ] + + with pytest.raises(AssertionError, match="runtime parity mismatch"): + compare(adk, codex, expected_usage=(10, 4)) + + +@pytest.mark.asyncio +async def test_harness_detects_span_divergence(parity_runner, compare) -> None: + """Spans are compared only when both arms captured them -- and then it bites.""" + adk, codex = await _both(parity_runner) + adk.span_names = frozenset({"invocation", "call_llm"}) + codex.span_names = frozenset({"invocation"}) + + with pytest.raises(AssertionError, match="call_llm"): + compare(adk, codex) + + +@pytest.mark.asyncio +async def test_harness_detects_declared_usage_shortfall(parity_runner, compare) -> None: + """``expected_usage`` pins the plan's exact total, not merely "non-zero".""" + adk, codex = await _both(parity_runner) + + with pytest.raises(AssertionError, match="plan declares"): + compare(adk, codex, expected_usage=(999, 999)) + + +@pytest.mark.asyncio +async def test_harness_detects_error_type_divergence(parity_runner, compare) -> None: + """An exception in one arm and not the other is itself a divergence.""" + adk, codex = await _both(parity_runner) + codex.error = RuntimeError("boom") + + with pytest.raises(AssertionError, match="error type"): + compare(adk, codex) + + +@pytest.mark.asyncio +async def test_unfaulted_run_compares_equal(parity_runner, compare) -> None: + """The control: without an injected fault the two arms agree. + + Without this, every ``pytest.raises`` above could be passing for the wrong + reason (a harness that always raises). + """ + adk, codex = await _both(parity_runner) + assert adk.error is None, adk.error + assert codex.error is None, codex.error + compare(adk, codex, expected_usage=(10, 4)) + + +# ------------------------------------------------- closed-allowlist guarding + + +def test_classifier_rejects_an_event_it_cannot_classify(event_classifier) -> None: + """An unclassifiable event raises rather than being silently dropped. + + This is what forces a human decision when the Codex SDK grows a new + notification type: the alternative -- dropping it -- would quietly shrink + the equivalence class. + """ + mystery = SimpleNamespace( + partial=False, + content=None, + error_code=None, + custom_metadata=None, + actions=None, + ) + with pytest.raises(AssertionError, match="unclassified event"): + event_classifier(mystery) + + +def test_classifier_rejects_an_unknown_codex_lifecycle_type(event_classifier) -> None: + mystery = SimpleNamespace( + partial=False, + content=None, + error_code=None, + custom_metadata={"codex_event_type": "brand_new_notification"}, + actions=None, + ) + with pytest.raises(AssertionError, match="unclassified codex lifecycle"): + event_classifier(mystery) + + +@pytest.mark.parametrize( + ("event", "expected"), + [ + pytest.param( + Event( + invocation_id="inv", + author="a", + content=types.Content(role="model", parts=[types.Part(text="hi")]), + ), + "text", + id="text", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + partial=True, + content=types.Content(role="model", parts=[types.Part(text="h")]), + ), + "delta", + id="delta", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + content=types.Content( + role="model", parts=[types.Part(text="why", thought=True)] + ), + ), + "thought", + id="thought", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall(id="c", name="t", args={}) + ) + ], + ), + ), + "function_call", + id="function_call", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + content=types.Content( + role="user", + parts=[ + types.Part( + function_response=types.FunctionResponse( + id="c", name="t", response={} + ) + ) + ], + ), + ), + "function_response", + id="function_response", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + custom_metadata={"codex_event_type": "turn_complete"}, + turn_complete=True, + ), + "codex_lifecycle", + id="lifecycle", + ), + pytest.param( + Event( + invocation_id="inv", + author="a", + error_code="backend", + error_message="nope", + ), + "error", + id="error", + ), + ], +) +def test_classifier_vocabulary(event_classifier, event, expected) -> None: + assert event_classifier(event) == expected + + +# ------------------------------------------- the backend records its inputs + + +@pytest.mark.asyncio +async def test_recorded_call_is_shaped_the_same_in_both_arms(parity_runner) -> None: + """The normalizer really does make two different protocols comparable. + + Without this, ``RecordedCall`` equality could be trivially satisfied by two + arms that both record nothing. + """ + adk, codex = await _both(parity_runner) + assert adk.calls and codex.calls + assert adk.calls[0].current_text == "do the thing" + assert codex.calls[0].current_text == "do the thing" + assert adk.calls[0].tool_names == ("record_fact",) + assert codex.calls[0].tool_names == ("record_fact",) + # The second request must show the model its own tool history. + assert adk.calls[1].tool_records == ( + ("function_call", "record_fact"), + ("function_response", "record_fact"), + ) + assert codex.calls[1].tool_records == adk.calls[1].tool_records diff --git a/tests/runtime/differential/test_runtime_parity.py b/tests/runtime/differential/test_runtime_parity.py new file mode 100644 index 000000000..194b832d7 --- /dev/null +++ b/tests/runtime/differential/test_runtime_parity.py @@ -0,0 +1,651 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Differential ("对拍") matrix: one config, two runtimes, one equivalence class. + +Each row runs the *same* agent configuration through ``runtime="adk"`` and +``runtime="codex"`` against the same scripted turn plan, then asserts the two +observable outcomes are equivalent. Rows are split three ways: + +``PARITY_ROWS`` + Must produce equivalent observations. A failure here is a real divergence. + +``XFAIL_ROWS`` + Divergences ``veadk.runtime.compat`` deliberately classifies as ``warn``: + accepted, documented, and *still asserted*, with ``strict=True`` so closing + the gap makes the suite fail until the marker is removed. Turning one of + these into a plain failure is a one-line change if you would rather see red. + +``ERROR_ROWS`` + Configurations ``compat`` classifies as ``error``. Here the contract is the + refusal itself: constructing the agent must raise a ``ValueError`` that + names the field, says what would silently break, and offers a way out. + +What is deliberately *excluded* from the equivalence class -- reasoning parts, +Codex lifecycle events, ``partial`` deltas, exact event counts, ids/timestamps +-- each carries a paired positive assertion at the bottom of this file. An +exclusion with no paired assertion is a hole, not a simplification. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable + +import pytest +from google.adk.agents.invocation_context import LlmCallsLimitExceededError +from google.adk.agents.run_config import RunConfig +from google.adk.examples.base_example_provider import BaseExampleProvider +from google.adk.examples.example import Example +from google.adk.planners.plan_re_act_planner import PlanReActPlanner +from google.adk.tools.long_running_tool import LongRunningFunctionTool +from google.genai import types +from pydantic import BaseModel + +from scripted_backend import Round, ScriptedBackend + +KB_MARKER = "differential-kb-marker" +EXAMPLE_MARKER = "differential-example-marker" + + +# ------------------------------------------------------------------- tools + + +def record_fact(fact: str) -> dict: + """Record a fact for later.""" + return {"stored": fact} + + +def lookup_city(city: str) -> dict: + """Look up a city.""" + return {"city": city, "population": 42} + + +def slow_approval(request: str) -> dict: + """A tool whose real work finishes out of band.""" + return {"ticket": request} + + +class _Examples(BaseExampleProvider): + def get_examples(self, query: str) -> list[Example]: + return [ + Example( + input=types.Content( + role="user", parts=[types.Part(text=f"{EXAMPLE_MARKER} question")] + ), + output=[ + types.Content( + role="model", + parts=[types.Part(text=f"{EXAMPLE_MARKER} answer")], + ) + ], + ) + ] + + +def _knowledgebase_tool() -> Any: + """A real ``LoadKnowledgebaseTool`` over a stand-in knowledge base. + + The row is about the *mechanism*: the knowledge base is wired in by + ``LoadKnowledgebaseTool.process_llm_request``, which the Codex runtime never + calls, so retrieval is never advertised to the model. + """ + from types import SimpleNamespace + + from veadk.tools.builtin_tools.load_knowledgebase import LoadKnowledgebaseTool + + return LoadKnowledgebaseTool( + knowledgebase=SimpleNamespace( + name=KB_MARKER, + description="Differential knowledge base.", + backend="fake", + enable_profile=False, + ) + ) + + +class _Answer(BaseModel): + answer: str + + +# --------------------------------------------------------------- row types + + +@dataclass(frozen=True) +class Row: + """One matrix row: a config, a plan, and what must hold about the pair.""" + + id: str + plan: tuple[Round, ...] + agent_kwargs: Callable[[ScriptedBackend], dict] | dict = field(default_factory=dict) + expected_usage: tuple[int, int] | None = None + run_config: Any = None + extra_assert: Callable[[Any, Any], None] | None = None + capture_spans: bool = False + expect_error: type[BaseException] | None = None + + +def _kwargs(**values: Any) -> Callable[[ScriptedBackend], dict]: + return lambda _backend: dict(values) + + +# ------------------------------------------------------------ extra asserts + + +def _assert_tool_history_replayed(adk, codex) -> None: + """The third request must show the model both completed tool round-trips.""" + want = ( + ("function_call", "record_fact"), + ("function_response", "record_fact"), + ("function_call", "lookup_city"), + ("function_response", "lookup_city"), + ) + assert adk.calls[2].tool_records == want, adk.calls[2].tool_records + assert codex.calls[2].tool_records == want, codex.calls[2].tool_records + + +def _assert_output_key_from_session(adk, codex) -> None: + """``output_key`` is read from the session service, not from the events. + + That is what a downstream ``SequentialAgent`` node actually sees. + """ + assert adk.session_state.get("answer") == "The sky is blue." + assert codex.session_state.get("answer") == "The sky is blue." + + +def _assert_output_key_joins_multi_text(adk, codex) -> None: + assert adk.session_state.get("answer") == "Part one. Part two." + assert codex.session_state.get("answer") == "Part one. Part two." + + +def _assert_temperature_reaches_backend(adk, codex) -> None: + assert adk.calls[0].temperature == 0.1 + assert codex.calls[0].temperature == 0.1 + assert adk.calls[0].max_output_tokens == 64 + assert codex.calls[0].max_output_tokens == 64 + + +def _assert_no_history_leaks(adk, codex) -> None: + assert adk.calls[0].history_texts == () + assert codex.calls[0].history_texts == () + + +def _assert_planner_instruction(adk, codex) -> None: + for arm in (adk, codex): + assert "/*PLANNING*/" in arm.calls[0].system_instruction, arm.arm + + +def _assert_example_reaches_prompt(adk, codex) -> None: + for arm in (adk, codex): + assert EXAMPLE_MARKER in arm.calls[0].system_instruction, arm.arm + + +def _assert_knowledgebase_reaches_prompt(adk, codex) -> None: + for arm in (adk, codex): + assert KB_MARKER in arm.calls[0].system_instruction, arm.arm + + +def _assert_transfer_tool_advertised(adk, codex) -> None: + for arm in (adk, codex): + assert "transfer_to_agent" in arm.calls[0].tool_names, arm.arm + + +def _assert_long_running_marked(adk, codex) -> None: + for arm in (adk, codex): + ids = set() + for event in arm.events: + ids |= set(getattr(event, "long_running_tool_ids", None) or ()) + assert ids, f"{arm.arm}: no event carried long_running_tool_ids" + + +def _assert_callbacks_applied(adk, codex) -> None: + for arm in (adk, codex): + assert arm.final_text.endswith("[after]"), (arm.arm, arm.final_text) + assert "[before]" in arm.calls[0].system_instruction, arm.arm + + +# ---------------------------------------------------------------- the plans + + +_TEXT_PLAN = (Round(text="The sky is blue.", usage=(10, 4)),) +_TOOL_PLAN = ( + Round(tool_calls=(("record_fact", {"fact": "sky is blue"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), +) +_TWO_TOOL_PLAN = ( + Round(tool_calls=(("record_fact", {"fact": "blue"}),), usage=(0, 0)), + Round(tool_calls=(("lookup_city", {"city": "Beijing"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), +) +_USAGE_PLAN = ( + Round(tool_calls=(("record_fact", {"fact": "blue"}),), usage=(11, 5)), + Round(text="The sky is blue.", usage=(7, 3)), +) + + +def _before_model(callback_context, llm_request): # noqa: ANN001 + llm_request.append_instructions(["[before]"]) + return None + + +def _after_model(callback_context, llm_response): # noqa: ANN001 + parts = list((llm_response.content.parts if llm_response.content else []) or []) + if parts and parts[-1].text: + parts[-1] = types.Part(text=parts[-1].text + "[after]") + llm_response.content = types.Content(role="model", parts=parts) + return llm_response + + +PARITY_ROWS: tuple[Row, ...] = ( + Row(id="baseline_text", plan=_TEXT_PLAN, expected_usage=(10, 4)), + Row( + id="single_tool", + plan=_TOOL_PLAN, + agent_kwargs=_kwargs(tools=[record_fact]), + expected_usage=(10, 4), + ), + Row( + id="two_tools_sequential", + plan=_TWO_TOOL_PLAN, + agent_kwargs=_kwargs(tools=[record_fact, lookup_city]), + expected_usage=(10, 4), + extra_assert=_assert_tool_history_replayed, + ), + Row( + id="output_key", + plan=_TOOL_PLAN, + agent_kwargs=_kwargs(tools=[record_fact], output_key="answer"), + expected_usage=(10, 4), + extra_assert=_assert_output_key_from_session, + ), + Row( + id="output_key_multi_text", + plan=(Round(texts=("Part one. ", "Part two."), usage=(10, 4)),), + agent_kwargs=_kwargs(output_key="answer"), + expected_usage=(10, 4), + extra_assert=_assert_output_key_joins_multi_text, + ), + Row( + id="before_after_model_callbacks", + plan=_TEXT_PLAN, + agent_kwargs=_kwargs( + before_model_callback=_before_model, after_model_callback=_after_model + ), + expected_usage=(10, 4), + extra_assert=_assert_callbacks_applied, + ), + Row( + id="long_running_tool", + plan=( + Round(tool_calls=(("slow_approval", {"request": "deploy"}),), usage=(0, 0)), + Round(text="The sky is blue.", usage=(10, 4)), + ), + agent_kwargs=_kwargs(tools=[LongRunningFunctionTool(func=slow_approval)]), + expected_usage=(10, 4), + extra_assert=_assert_long_running_marked, + ), + Row( + id="tracing_spans", + plan=_TEXT_PLAN, + expected_usage=(10, 4), + capture_spans=True, + ), + Row( + id="usage_accounting", + plan=_USAGE_PLAN, + agent_kwargs=_kwargs(tools=[record_fact]), + expected_usage=(18, 8), + ), + Row( + id="max_llm_calls", + plan=_TWO_TOOL_PLAN, + agent_kwargs=_kwargs(tools=[record_fact, lookup_city]), + run_config=RunConfig(max_llm_calls=1), + expect_error=LlmCallsLimitExceededError, + ), +) + +#: Divergences ``compat.py`` classifies as ``warn``: real, accepted, asserted. +#: ``strict=True`` makes closing one of them fail the suite until the marker is +#: removed, so an accepted gap can never quietly become an unnoticed feature. +XFAIL_ROWS: tuple[tuple[Row, str], ...] = ( + ( + Row( + id="example_store", + plan=_TEXT_PLAN, + agent_kwargs=_kwargs(example_store=_Examples()), + expected_usage=(10, 4), + extra_assert=_assert_example_reaches_prompt, + ), + "codex never calls ExampleTool.process_llm_request, so few-shot " + "examples never reach the model (compat.py: warn)", + ), + ( + Row( + id="knowledgebase", + plan=_TEXT_PLAN, + agent_kwargs=lambda _b: {"tools": [_knowledgebase_tool()]}, + expected_usage=(10, 4), + extra_assert=_assert_knowledgebase_reaches_prompt, + ), + "codex never calls LoadKnowledgebaseTool.process_llm_request, so the " + "model is never told the knowledge base exists (compat.py: warn)", + ), +) + +#: Configurations ``compat.py`` refuses outright. The contract is the message. +ERROR_ROWS: tuple[tuple[str, dict, tuple[str, ...]], ...] = ( + ( + "model_object", + {"model": "sentinel-base-llm"}, + ("Agent(model=", "model_name", "runtime='adk'"), + ), + ( + "output_schema", + {"output_schema": _Answer}, + ("output_schema", "runtime='adk'"), + ), + ( + "generate_content_config", + { + "generate_content_config": types.GenerateContentConfig( + temperature=0.1, max_output_tokens=64, stop_sequences=["STOP"] + ) + }, + ("generate_content_config", "temperature", "silently dropped"), + ), + ( + "include_contents_none", + {"include_contents": "none"}, + ("include_contents", "history", "runtime='adk'"), + ), + ("planner", {"planner": PlanReActPlanner()}, ("planner", "runtime='adk'")), +) + + +# ------------------------------------------------------------------- tests + + +async def _run_pair(parity_runner, row: Row): + adk = await parity_runner.run( + "adk", + plan=row.plan, + agent_kwargs=row.agent_kwargs, + run_config=row.run_config, + capture_spans=row.capture_spans, + ) + codex = await parity_runner.run( + "codex", + plan=row.plan, + agent_kwargs=row.agent_kwargs, + run_config=row.run_config, + capture_spans=row.capture_spans, + ) + return adk, codex + + +@pytest.mark.asyncio +@pytest.mark.parametrize("row", PARITY_ROWS, ids=lambda row: row.id) +async def test_runtime_parity(parity_runner, compare, row: Row) -> None: + adk, codex = await _run_pair(parity_runner, row) + if row.expect_error is not None: + assert isinstance(adk.error, row.expect_error), adk.error + assert isinstance(codex.error, row.expect_error), codex.error + return + compare(adk, codex, expected_usage=row.expected_usage) + if row.extra_assert is not None: + row.extra_assert(adk, codex) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "row", + [ + pytest.param( + row, id=row.id, marks=pytest.mark.xfail(strict=True, reason=reason) + ) + for row, reason in XFAIL_ROWS + ], +) +async def test_runtime_parity_accepted_divergence( + parity_runner, compare, row: Row +) -> None: + """Same assertions as :func:`test_runtime_parity`, for known ``warn`` gaps.""" + adk, codex = await _run_pair(parity_runner, row) + compare(adk, codex, expected_usage=row.expected_usage) + if row.extra_assert is not None: + row.extra_assert(adk, codex) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("row_id", "kwargs", "fragments"), ERROR_ROWS, ids=[r[0] for r in ERROR_ROWS] +) +async def test_unsupported_config_fails_fast_and_actionably( + parity_runner, row_id: str, kwargs: dict, fragments: tuple[str, ...] +) -> None: + """``compat.py`` must refuse, name the field, and offer a way out. + + A silent no-op is the failure mode this whole suite exists to prevent; a + refusal is only useful if its message tells the caller what to do instead. + """ + from veadk import Agent + + resolved = dict(kwargs) + if resolved.get("model") == "sentinel-base-llm": + resolved["model"] = ScriptedBackend([Round(text="x")]).as_base_llm() + if resolved.get("sub_agents") == "sentinel-sub-agents": + resolved["sub_agents"] = [ + Agent(name="specialist", model_name="scripted-model", model_api_key="k") + ] + + with pytest.raises(ValueError) as excinfo: + Agent( + name="parity_agent", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + **resolved, + ) + message = str(excinfo.value) + for fragment in fragments: + assert fragment in message, f"{row_id}: {fragment!r} missing from {message!r}" + + +def test_output_schema_rule_stays_an_error() -> None: + """Pin the invariant that makes ``output_state``'s schema branch dead code. + + ``maybe_save_output_to_state`` guards its ``output_schema`` branch instead + of raising, and documents itself as unreachable because this rule refuses + the configuration long before an event reaches it. Demoting the rule to + ``"warn"`` would quietly revive that branch against a model nobody + constrained -- so the demotion has to be a deliberate act that fails here, + on its own diff, rather than a silent behaviour change discovered later in + someone's session state. + + If you are here because this test failed: re-read + ``veadk/runtime/output_state.py``'s docstring before changing the rule. + """ + from veadk.runtime import compat + + rule = next( + (r for r in compat.SUPPORT_RULES if r.field == "output_schema"), + None, + ) + assert rule is not None, "the output_schema support rule was removed" + assert rule.policy == "error", ( + "output_schema was demoted to " + f"{rule.policy!r}; veadk/runtime/output_state.py assumes it stays " + "'error' and its schema branch is documented as unreachable." + ) + + +# -------------------------------------------------- paired positive asserts +# +# Every entry in the excluded set above is only safe to exclude because one of +# these pins the property that made it safe to ignore. + + +@pytest.mark.asyncio +async def test_excluded_reasoning_parts_are_thoughts_and_never_reach_output_key( + parity_runner, +) -> None: + """Paired with: reasoning/thought parts are excluded from the comparison.""" + from google.adk.events.event import Event + + from veadk.runtime.codex.translate import item_to_events + from veadk.runtime.output_state import maybe_save_output_to_state + + events = item_to_events( + {"id": "r1", "type": "reasoning", "summary": [{"text": "thinking hard"}]}, + "parity_agent", + "inv", + ) + assert events, "a reasoning item must still be observable" + part = events[0].content.parts[0] + assert part.thought is True + assert part.text == "thinking hard" + + from types import SimpleNamespace + + agent = SimpleNamespace( + name="parity_agent", output_key="answer", output_schema=None + ) + for event in events: + assert isinstance(event, Event) + maybe_save_output_to_state(agent, event) + assert event.actions.state_delta == {} + + +@pytest.mark.asyncio +async def test_excluded_codex_lifecycle_events_exist_and_carry_turn_id( + parity_runner, +) -> None: + """Paired with: Codex lifecycle events are excluded from the comparison.""" + codex = await parity_runner.run("codex", plan=_TEXT_PLAN) + assert codex.error is None, codex.error + + lifecycle = [ + event + for event in codex.events + if (event.custom_metadata or {}).get("codex_event_type") + ] + assert lifecycle, "the codex arm produced no lifecycle events at all" + by_type = { + (event.custom_metadata or {})["codex_event_type"]: event for event in lifecycle + } + assert "turn_started" in by_type + assert "turn_complete" in by_type + assert by_type["turn_started"].custom_metadata["turn_id"] == "turn-1" + assert by_type["turn_complete"].custom_metadata["turn_id"] == "turn-1" + + +@pytest.mark.asyncio +async def test_excluded_partial_deltas_precede_their_completed_item( + parity_runner, +) -> None: + """Paired with: ``partial=True`` deltas are excluded from the comparison.""" + codex = await parity_runner.run("codex", plan=_TEXT_PLAN) + assert codex.error is None, codex.error + + delta_at = next( + ( + index + for index, event in enumerate(codex.events) + if (event.custom_metadata or {}).get("codex_event_type") == "message_delta" + ), + None, + ) + completed_at = next( + ( + index + for index, event in enumerate(codex.events) + if (event.custom_metadata or {}).get("codex_event_type") == "item_completed" + ), + None, + ) + assert delta_at is not None, "no streaming delta was emitted" + assert completed_at is not None, "no completed item was emitted" + assert delta_at < completed_at + assert codex.events[delta_at].partial is True + + +@pytest.mark.asyncio +async def test_codex_arm_really_drove_the_shim_over_http(parity_runner) -> None: + """The fake is not allowed to shortcut the shim. + + If this ever passes with an empty request log, the codex arm has silently + stopped exercising ``proxy._synth_sse`` and every row above is testing a + much smaller system than it claims to. + """ + import fake_codex_sdk + + codex = await parity_runner.run( + "codex", plan=_TOOL_PLAN, agent_kwargs=_kwargs(tools=[record_fact]) + ) + assert codex.error is None, codex.error + assert fake_codex_sdk.REQUEST_LOG, "the codex arm never POSTed to the shim" + assert all(body["stream"] is True for body in fake_codex_sdk.REQUEST_LOG) + + +def test_plain_codex_agent_constructs_without_tripping_the_model_rule() -> None: + """The fail-fast layer must not fire on an ordinary codex agent. + + ``model_post_init`` assigns ``self.model`` itself, so a rule keyed on raw + ``model_fields_set`` would make the ``model`` ERROR fire for *every* codex + agent. ``Agent`` snapshots ``frozenset(self.model_fields_set)`` into + ``_veadk_explicit_fields`` at the top of ``model_post_init`` to distinguish + "the caller passed it" from "we assigned it". + """ + from veadk import Agent + from veadk.runtime.compat import explicit_fields + + agent = Agent( + name="plain_codex_agent", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + ) + + assert agent.model, "model_post_init should still have built a client" + explicit = explicit_fields(agent) + assert "model" not in explicit, explicit + assert "model_extra_config" not in explicit, explicit + assert "model_name" in explicit + + +def test_explicit_field_snapshot_survives_clone() -> None: + """``BaseAgent.clone()`` re-``setattr``s list fields, widening fields_set. + + Without the snapshot surviving the clone, a per-request clone of a valid + codex agent would start failing the ``model`` / ``model_extra_config`` + rules that the original passed. + """ + from veadk import Agent + from veadk.runtime.compat import check_agent_runtime_support, explicit_fields + + agent = Agent( + name="clonable_codex_agent", + model_name="scripted-model", + model_api_base="https://backend.invalid/v1", + model_api_key="backend-key", + runtime="codex", + ) + clone = agent.clone() + + assert explicit_fields(clone) == explicit_fields(agent) + # The real contract: the clone still validates. + check_agent_runtime_support(clone, "codex") diff --git a/tests/runtime/piagent/test_piagent_runtime.py b/tests/runtime/piagent/test_piagent_runtime.py index 11f9a3a19..24fdafdd3 100644 --- a/tests/runtime/piagent/test_piagent_runtime.py +++ b/tests/runtime/piagent/test_piagent_runtime.py @@ -1974,7 +1974,14 @@ async def test_piagent_runtime_loads_and_cleans_materialized_skills( argv = json.loads(argv_path.read_text(encoding="utf-8")) skill_path = Path(argv[argv.index("--skill") + 1]) - assert events == [] + # This fake Pi answers nothing at all, so the turn's only event is the + # contentless merged response that carries its bookkeeping (token usage, any + # `state_delta` a model callback wrote). It is emitted rather than dropped + # because there is no durable event to fold it onto, and being contentless + # it cannot clobber `output_key` or the evaluated answer -- both of which + # require content. See `test_piagent_turn_contract.py` for the tool-only + # turn, where the fold does have a target. + assert [e for e in events if e.content and e.content.parts] == [] assert "--no-skills" in argv assert skill_path.name == "demo-skill" assert not skill_path.exists() diff --git a/tests/runtime/piagent/test_piagent_runtime_smoke.py b/tests/runtime/piagent/test_piagent_runtime_smoke.py index 7745cc6a0..a7e71b143 100644 --- a/tests/runtime/piagent/test_piagent_runtime_smoke.py +++ b/tests/runtime/piagent/test_piagent_runtime_smoke.py @@ -37,8 +37,11 @@ from veadk import Agent, Runner -def pytest_configure(config): - config.addinivalue_line("markers", "piagent_smoke: real Pi binary/model smoke test") +# NOTE: `pytest_configure` is only collected from `conftest.py` and plugins, so +# defining it here would be dead code. The `piagent_smoke` marker is registered +# in `pytest.ini` instead. (It previously appeared to work only because +# `filterwarnings = ignore::UserWarning` swallowed the resulting +# `PytestUnknownMarkWarning`.) @pytest.mark.piagent_smoke diff --git a/tests/runtime/piagent/test_piagent_turn_contract.py b/tests/runtime/piagent/test_piagent_turn_contract.py new file mode 100644 index 000000000..50e862045 --- /dev/null +++ b/tests/runtime/piagent/test_piagent_turn_contract.py @@ -0,0 +1,669 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whole-turn contract tests for the Pi runtime. + +``test_piagent_runtime.py`` drives single-round Pi streams. The bugs this file +covers are all *multi-round*: they need a turn where the model speaks twice, and +they were invisible to any test that could only express one round. + +The context object matters as much as the stream. ``_fake_ctx`` in the sibling +file is a bare ``SimpleNamespace`` with no ``increment_llm_call_count``, and the +runtime reaches that hook through ``getattr(ctx, ..., None)`` -- so a budget test +written against it passes no matter what the runtime does. :func:`_counting_ctx` +supplies a real counter with ADK's semantics instead. +""" + +from __future__ import annotations + +import json +import stat +from types import SimpleNamespace + +import pytest +from google.adk.agents.invocation_context import LlmCallsLimitExceededError +from google.adk.events.event import Event +from google.genai import types + +from veadk import Agent +from veadk.runtime.piagent.runtime import PiAgentRuntime +from veadk.runtime.piagent.translate import PiEventTranslator + + +def _user_event(text: str) -> Event: + return Event( + invocation_id="inv-user", + author="user", + content=types.Content(role="user", parts=[types.Part(text=text)]), + ) + + +def _counting_ctx(*events: Event, max_llm_calls: int = 0): + """A context whose ``increment_llm_call_count`` behaves like ADK's. + + ADK raises once the count *exceeds* the budget, and that raise is what + ``RunConfig.max_llm_calls`` is made of. A ``SimpleNamespace`` without this + method makes the runtime's ``getattr`` guard swallow every charge, so a test + using one cannot tell enforcement from its absence. + """ + state: dict[str, int] = {"calls": 0} + + def increment_llm_call_count() -> None: + state["calls"] += 1 + if max_llm_calls and state["calls"] > max_llm_calls: + raise LlmCallsLimitExceededError( + f"Max number of llm calls limit of {max_llm_calls} exceeded" + ) + + ctx = SimpleNamespace( + invocation_id="inv-1", + session=SimpleNamespace(events=list(events), state={}), + increment_llm_call_count=increment_llm_call_count, + ) + ctx.llm_call_state = state + return ctx + + +def _make_pi_emitting(tmp_path, lines: list[dict]): + """A fake Pi binary that replays ``lines`` as NDJSON for one prompt.""" + path = tmp_path / "pi" + payload = json.dumps(lines) + path.write_text( + f"""#!/usr/bin/env python3 +import json +import os +import sys + +agent_dir = os.environ.get("PI_CODING_AGENT_DIR") +assert agent_dir, "PI_CODING_AGENT_DIR missing" + +LINES = json.loads({payload!r}) + +for raw in sys.stdin: + command = json.loads(raw) + if command.get("type") == "prompt": + print(json.dumps({{ + "id": command.get("id"), + "type": "response", + "command": "prompt", + "success": True, + }}), flush=True) + for line in LINES: + print(json.dumps(line), flush=True) + break +""", + encoding="utf-8", + ) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + +def _assistant_message(text: str, usage: dict | None = None) -> dict: + message = { + "role": "assistant", + "content": [{"type": "text", "text": text}], + } + if usage is not None: + message["usage"] = usage + return message + + +#: A turn where the model writes a preamble alongside its tool call and only +#: answers in round two -- routine model behaviour, and the shape that the +#: `emitted_text` latch got wrong. +_PREAMBLE_THEN_ANSWER = [ + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "let me check"}, + }, + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "get_weather", + "args": {"city": "Beijing"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "get_weather", + "result": {"weather": "sunny"}, + }, + # Round one closes, re-announcing the preamble the tool-call event carried. + {"type": "message_end", "message": _assistant_message("let me check")}, + # Round two: the answer. + {"type": "message_end", "message": _assistant_message("it is sunny")}, + {"type": "agent_settled"}, +] + + +def _agent(**kwargs): + return Agent( + name="assistant", + instruction="Answer briefly.", + model_name="model-a", + model_api_base="https://ark.example.com/api/v3/", + model_api_key="test-key", + model_api_key_name="", + runtime="piagent", + **kwargs, + ) + + +# ------------------------------------------------- the first-round-wins blocker + + +@pytest.mark.asyncio +async def test_answer_wins_over_a_tool_call_preamble(tmp_path, monkeypatch) -> None: + """The round that actually answered must be the turn's answer. + + ``_flush_events`` used to gate on a boolean ``emitted_text`` latch, so the + first assistant message carrying visible text won for the whole invocation. + On any turn whose tool call has a text preamble the preamble became the + answer and the answering round was dropped -- no error, no log, just the + wrong reply. Buffering alone does not fix it: the buffer simply receives the + preamble instead. + """ + binary = _make_pi_emitting(tmp_path, _PREAMBLE_THEN_ANSWER) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + ctx = _counting_ctx(_user_event("weather?")) + events = [e async for e in PiAgentRuntime().run_async(_agent(), ctx)] + + finals = [ + e for e in events if e.is_final_response() and e.content and e.content.parts + ] + assert len(finals) == 1, ( + "a turn must produce exactly one final response; got " + f"{[[p.text for p in e.content.parts] for e in finals]}" + ) + text = "".join(p.text or "" for p in finals[0].content.parts) + assert "it is sunny" in text, ( + f"the answering round was dropped; the turn answered {text!r}" + ) + assert text.strip() == "it is sunny", ( + f"the preamble leaked into the turn's answer: {text!r}" + ) + + +# ------------------------------------------------------ the new runtime plumbing + + +@pytest.mark.asyncio +async def test_max_llm_calls_is_charged_per_model_call(tmp_path, monkeypatch) -> None: + """Every completed Pi model call must charge ADK's budget. + + Enforcement is one call late by design: Pi owns its loop inside the binary + and only reports a call once it has finished, so the invocation aborts just + *past* the limit rather than just short of it. What must not happen is the + budget never being charged at all -- which is what a context without + ``increment_llm_call_count`` hides. + """ + binary = _make_pi_emitting(tmp_path, _PREAMBLE_THEN_ANSWER) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + ctx = _counting_ctx(_user_event("weather?")) + _events = [e async for e in PiAgentRuntime().run_async(_agent(), ctx)] + + assert ctx.llm_call_state["calls"] == 2, ( + "the turn made two backend model calls (two assistant message_end " + f"events); ADK's budget was charged {ctx.llm_call_state['calls']} times" + ) + + +@pytest.mark.asyncio +async def test_max_llm_calls_aborts_the_invocation(tmp_path, monkeypatch) -> None: + """An exhausted budget must abort rather than be swallowed. + + ``LlmCallsLimitExceededError`` is re-raised rather than routed through + ``on_model_error``, matching ADK: ``Runner`` handles it itself. + """ + binary = _make_pi_emitting(tmp_path, _PREAMBLE_THEN_ANSWER) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + ctx = _counting_ctx(_user_event("weather?"), max_llm_calls=1) + + with pytest.raises(LlmCallsLimitExceededError): + async for _event in PiAgentRuntime().run_async(_agent(), ctx): + pass + + +@pytest.mark.asyncio +async def test_turn_reports_accumulated_token_usage(tmp_path, monkeypatch) -> None: + """One usage carrier per turn, summed across rounds. + + Consumers add ``usage_metadata`` up across events without deduplicating, so + a turn must attach it exactly once. Pi reports Anthropic-style *disjoint* + prompt counters -- ``input`` excludes cached tokens, which arrive separately + as ``cacheRead``/``cacheWrite`` -- whereas genai's ``prompt_token_count`` is + the whole prompt with ``cached_content_token_count`` a subset of it. This is + deliberately not the codex mapping; do not share an assertion helper. + """ + lines = [ + { + "type": "message_end", + "message": _assistant_message( + "round one", + usage={"input": 10, "output": 4, "cacheRead": 3, "cacheWrite": 2}, + ), + }, + { + "type": "message_end", + "message": _assistant_message("round two", usage={"input": 5, "output": 6}), + }, + {"type": "agent_settled"}, + ] + binary = _make_pi_emitting(tmp_path, lines) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + ctx = _counting_ctx(_user_event("hi")) + events = [e async for e in PiAgentRuntime().run_async(_agent(), ctx)] + + carriers = [e for e in events if getattr(e, "usage_metadata", None) is not None] + assert len(carriers) == 1, ( + f"expected exactly one usage carrier per turn, got {len(carriers)}" + ) + usage = carriers[0].usage_metadata + # prompt <- input + cacheRead + cacheWrite = (10+3+2) + 5 + assert usage.prompt_token_count == 20, usage + # candidates <- output = 4 + 6 + assert usage.candidates_token_count == 10, usage + assert usage.cached_content_token_count == 3, usage + assert usage.total_token_count == 30, usage + # `reasoning` is a subset of `output` in Pi's accounting, so mapping it onto + # genai's disjoint `thoughts_token_count` would double-count it. + assert usage.thoughts_token_count is None, usage + + +def test_reasoning_tokens_are_never_mapped_to_thoughts() -> None: + """Unit-level guard for the one mapping that must stay unmapped.""" + translator = PiEventTranslator(author="assistant", invocation_id="inv-1") + translator.event_to_adk_events( + { + "type": "message_end", + "message": _assistant_message( + "hi", usage={"input": 8, "output": 5, "reasoning": 4} + ), + } + ) + usage = translator.build_turn_usage_metadata() + assert usage is not None + assert usage.prompt_token_count == 8, usage + assert usage.candidates_token_count == 5, usage + assert usage.thoughts_token_count is None, ( + "Pi's `reasoning` is a subset of `output`; genai treats thoughts as " + "disjoint from candidates, so mapping it double-counts" + ) + + +@pytest.mark.asyncio +async def test_turn_emits_an_indexable_call_llm_span(tmp_path, monkeypatch) -> None: + """A Pi turn must open the ``call_llm`` span VeADK's telemetry keys off. + + ``_InMemoryExporter`` indexes a session only from spans literally named + ``call_llm`` carrying ``gen_ai.session.id``; ADK opens that span inside the + LLM flow this runtime replaces, so without one every Pi trace dump is ``[]`` + and any evaluation built from one raises. + + Driven through a real ``Runner`` rather than :func:`_counting_ctx`: the + session index is the property that matters, and a ``SimpleNamespace`` + context has no session for the telemetry writer to key off (it fails soft, + logging ``piagent_trace_call_llm_failed``, which would leave this test + asserting only that a bare unattributed span exists). + """ + import uuid + + from google.adk.runners import Runner + from google.adk.sessions.in_memory_session_service import InMemorySessionService + + from veadk.tracing.telemetry.opentelemetry_tracer import OpentelemetryTracer + + binary = _make_pi_emitting( + tmp_path, + [ + { + "type": "message_end", + "message": _assistant_message("hi", usage={"input": 11, "output": 7}), + }, + {"type": "agent_settled"}, + ], + ) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + # Attaches to whichever provider is already active; deliberately no global + # TracerProvider swap (ADK's module-level tracers memoize the first real + # provider they resolve, so replacing it makes later spans vanish). + tracer = OpentelemetryTracer(exporters=[]) + exporter = tracer._inmemory_exporter._exporter + before = len(exporter._spans) + + agent = _agent(tracers=[tracer]) + session_id = f"session-{uuid.uuid4().hex[:8]}" + session_service = InMemorySessionService() + await session_service.create_session( + app_name="pi", user_id="user", session_id=session_id + ) + runner = Runner(app_name="pi", agent=agent, session_service=session_service) + + events = [ + event + async for event in runner.run_async( + user_id="user", + session_id=session_id, + new_message=types.Content(role="user", parts=[types.Part(text="hi")]), + ) + ] + assert events, "the pi run produced no events at all" + + spans = exporter._spans[before:] + call_llm = [s for s in spans if s.name == "call_llm"] + assert call_llm, ( + f"no call_llm span for a Pi turn: {sorted({s.name for s in spans})}" + ) + + attributes = dict(call_llm[0].attributes or {}) + assert attributes.get("gen_ai.session.id") == session_id, attributes + + # The property the exporter's session index -- and therefore every trace + # dump and every evaluation built from one -- actually depends on. + assert exporter.get_finished_spans(session_id), ( + "get_finished_spans() is empty, so OpentelemetryTracer.dump() would " + "write [] and base_evaluator.build_eval_set would raise" + ) + + +# ------------------------------------------------------------ the tool-only turn + + +#: A turn that ends in tool work and never speaks: the round's assistant message +#: carries only a `tool_use` block, so no `message_end` ever produces text. The +#: merged response for such a turn has no content -- but it still carries the +#: turn's `usage_metadata` and whatever `state_delta` a model callback wrote. +_TOOL_ONLY_TURN = [ + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "write_file", + "args": {"path": "notes.md"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "write_file", + "result": {"ok": True}, + }, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "tool_use", "id": "call-1", "name": "write_file"}], + "usage": {"input": 9, "output": 3}, + }, + }, + {"type": "agent_settled"}, +] + + +@pytest.mark.asyncio +async def test_tool_only_turn_keeps_callback_state_and_usage( + tmp_path, monkeypatch +) -> None: + """A text-less turn must not silently discard its own bookkeeping. + + ``run_before_model_callbacks``/``run_after_model_callbacks`` build their + ``CallbackContext`` over ``model_response_event.actions``, so a callback's + ``callback_context.state[...]`` writes land on the merged event's + ``state_delta``, and ``llm_response_to_event`` attaches the turn's + ``usage_metadata`` to the same event. Dropping that event whole -- which a + bare ``if event.content and event.content.parts`` guard does -- throws both + away without a word. + + Emitting it instead is not the alternative: a contentless, tool-free, + non-partial event is a final response by ADK's definition, so it would read + as the turn's answer. Nor does ``partial=True`` rescue it, since partial + events are never persisted. The bookkeeping is therefore folded onto the + last durable event the turn actually emitted. + """ + written: dict[str, str] = {} + + def after_model_callback(callback_context, llm_response): + callback_context.state["pi_last_turn"] = "tool-only" + written["ran"] = "yes" + return None + + binary = _make_pi_emitting(tmp_path, _TOOL_ONLY_TURN) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + agent = _agent(after_model_callback=after_model_callback) + ctx = _counting_ctx(_user_event("write the notes")) + events = [e async for e in PiAgentRuntime().run_async(agent, ctx)] + + assert written.get("ran") == "yes", "the after-model callback never ran" + assert events, "a tool-only turn emitted nothing at all" + + empty_finals = [ + e + for e in events + if e.is_final_response() and not (e.content and e.content.parts) + ] + assert not empty_finals, ( + "a contentless event was emitted; `Event.is_final_response()` is True " + "for it, so it reads as the turn's answer" + ) + + state_carriers = [e for e in events if e.actions.state_delta] + assert len(state_carriers) == 1, ( + "the callback's state write must survive on exactly one emitted event; " + f"found {[dict(e.actions.state_delta) for e in events]}" + ) + assert state_carriers[0].actions.state_delta["pi_last_turn"] == "tool-only" + + usage_carriers = [e for e in events if getattr(e, "usage_metadata", None)] + assert len(usage_carriers) == 1, ( + "the turn's token usage must survive on exactly one emitted event; " + f"got {len(usage_carriers)} carriers" + ) + assert usage_carriers[0].usage_metadata.prompt_token_count == 9 + assert usage_carriers[0].usage_metadata.candidates_token_count == 3 + + +@pytest.mark.asyncio +async def test_partials_are_not_stalled_behind_the_held_back_event( + tmp_path, monkeypatch +) -> None: + """Holding a durable event back must not stall the live stream. + + The last durable event is withheld so a text-less turn's bookkeeping has + somewhere to land. Parking the partials that follow it behind that event -- + to keep one global order -- stalls the stream for the rest of the turn: a + long command's output and the final answer's deltas all arrive *after* the + last durable event, so a user would see nothing until the Pi stream ended. + + Overtaking is safe precisely because partials are never persisted + (``BaseSessionService.append_event`` returns early on ``event.partial``), so + only the order among durable events is observable in session history, and + that order is unchanged. + """ + lines = [ + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "bash", + "args": {"command": "ls"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "bash", + "result": {"stdout": "notes.md"}, + }, + # Round two streams its answer *after* the last durable event. + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "one file"}, + }, + {"type": "message_end", "message": _assistant_message("one file")}, + {"type": "agent_settled"}, + ] + binary = _make_pi_emitting(tmp_path, lines) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + ctx = _counting_ctx(_user_event("what is in the dir?")) + events = [e async for e in PiAgentRuntime().run_async(_agent(), ctx)] + + def _index(predicate) -> int: + for index, event in enumerate(events): + if predicate(event): + return index + raise AssertionError(f"no event matched among {len(events)} events") + + partial_at = _index(lambda e: e.partial) + response_at = _index(lambda e: e.get_function_responses()) + assert partial_at < response_at, ( + "the streamed delta was delivered only after the held-back tool " + "response, so the live stream stalled for the rest of the turn" + ) + + # The durable order itself is untouched: the call still precedes its result. + call_at = _index(lambda e: e.get_function_calls()) + assert call_at < response_at + + finals = [ + e for e in events if e.is_final_response() and e.content and e.content.parts + ] + assert len(finals) == 1 + assert "".join(p.text or "" for p in finals[0].content.parts) == "one file" + + +# ------------------------------------------------------- the text-keyed dedup + + +#: The final round answers with text byte-identical to the preamble its own tool +#: call already carried -- a model that says "Done." beside the tool call and +#: "Done." again once the tool returned. Rare, but nothing prevents it. +_IDENTICAL_PREAMBLE_AND_ANSWER = [ + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "Done."}, + }, + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "write_file", + "args": {"path": "notes.md"}, + }, + { + "type": "tool_execution_end", + "toolCallId": "call-1", + "toolName": "write_file", + "result": {"ok": True}, + }, + # Round one closes, re-announcing the preamble the tool-call event carried. + {"type": "message_end", "message": _assistant_message("Done.")}, + # Round two: a *new* assistant message that happens to repeat the words. + {"type": "message_end", "message": _assistant_message("Done.")}, + {"type": "agent_settled"}, +] + + +@pytest.mark.asyncio +async def test_answer_survives_matching_an_earlier_preamble( + tmp_path, monkeypatch +) -> None: + """Dedup must suppress a re-announcement, never a new round's answer. + + Suppression used to be keyed on "has this exact text been emitted at any + point in the turn". That is right for a replay, but a round-closing + ``message_end`` only ever repeats the preamble *its own round* parked on a + tool-call event -- so keying it on the whole history silently dropped an + answering round whose text matched an earlier preamble. With no final text + event left, the merged response had no content, no final response was + emitted, and ``output_key`` was never written for the turn. + """ + from veadk.runtime.output_state import maybe_save_output_to_state + + binary = _make_pi_emitting(tmp_path, _IDENTICAL_PREAMBLE_AND_ANSWER) + monkeypatch.setenv("PIAGENT_BINARY", str(binary)) + monkeypatch.setenv("PIAGENT_AGENT_DIR", str(tmp_path / "agent-home")) + + agent = _agent(output_key="answer") + ctx = _counting_ctx(_user_event("write the notes")) + events = [e async for e in PiAgentRuntime().run_async(agent, ctx)] + + finals = [ + e for e in events if e.is_final_response() and e.content and e.content.parts + ] + assert len(finals) == 1, ( + "the answering round was dropped because its text matched the " + f"preamble; final responses: {len(finals)}" + ) + assert "".join(p.text or "" for p in finals[0].content.parts) == "Done." + + # `Agent._run_async_impl` runs exactly this over every runtime event; with + # the answer suppressed there is no event for it to write from. + saved = {} + for event in events: + maybe_save_output_to_state(agent, event) + saved.update(event.actions.state_delta) + assert saved.get("answer") == "Done.", ( + f"output_key was never written for the turn; state_delta: {saved}" + ) + + +def test_end_of_turn_replay_is_still_suppressed() -> None: + """The dedup the fix must not lose: a terminal replay of emitted text. + + ``agent_end`` re-announces the last assistant message wholesale once the + turn is over. Nothing new can arrive after it, so text it repeats is always + a replay -- and re-emitting it would both duplicate content already + persisted on a tool-call event and turn a preamble into the turn's answer. + """ + translator = PiEventTranslator(author="assistant", invocation_id="inv-1") + + translator.event_to_adk_events( + { + "type": "message_update", + "assistantMessageEvent": {"type": "text_delta", "delta": "on it"}, + } + ) + call = translator.event_to_adk_events( + { + "type": "tool_execution_start", + "toolCallId": "call-1", + "toolName": "write_file", + "args": {}, + } + ) + assert call[0].content.parts[0].text == "on it" + + # The round closes by repeating the preamble the tool-call event carried. + assert ( + translator.event_to_adk_events( + {"type": "message_end", "message": _assistant_message("on it")} + ) + == [] + ) + # ...and so does the end-of-turn replay, from a different code path. + assert ( + translator.event_to_adk_events( + {"type": "agent_end", "messages": [_assistant_message("on it")]} + ) + == [] + ) diff --git a/tests/runtime/test_output_state.py b/tests/runtime/test_output_state.py index 13fbe8e3a..c359f815a 100644 --- a/tests/runtime/test_output_state.py +++ b/tests/runtime/test_output_state.py @@ -21,7 +21,6 @@ from google.adk.runners import Runner from google.adk.sessions.in_memory_session_service import InMemorySessionService from google.genai import types -from pydantic import BaseModel from veadk import Agent from veadk.agents.sequential_agent import SequentialAgent @@ -112,26 +111,75 @@ def test_maybe_save_output_to_state_ignores_non_final_model_text(event: Event) - assert event.actions.state_delta == {} -def test_maybe_save_output_to_state_validates_output_schema() -> None: - class Result(BaseModel): - answer: str +def _codex_shaped_turn(author: str = "worker") -> list[Event]: + """A realistic external-harness turn, not a single perfect event. - agent = SimpleNamespace(name="agent", output_key="result", output_schema=Result) - event = _text_event('{"answer": "done"}') - - maybe_save_output_to_state(agent, event) - - assert event.actions.state_delta == {"result": {"answer": "done"}} + The previous version of this test yielded exactly one text event, which is + the one shape no real Codex turn ever has. A real turn interleaves + reasoning, tool lifecycle, streaming deltas and a contentless completion + marker around the answer -- and every one of those is a chance to write the + wrong thing into ``output_key``. + """ + reasoning = Event( + invocation_id="inv-1", + author=author, + content=types.Content( + role="model", + parts=[types.Part(text="the user wants a summary", thought=True)], + ), + custom_metadata={"codex_event_type": "item_completed"}, + ) + tool_call = Event( + invocation_id="inv-1", + author=author, + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + id="call-1", name="exec_command", args={"command": "ls"} + ) + ) + ], + ), + custom_metadata={"codex_event_type": "item_started"}, + ) + tool_response = Event( + invocation_id="inv-1", + author=author, + content=types.Content( + role="user", + parts=[ + types.Part( + function_response=types.FunctionResponse( + id="call-1", name="exec_command", response={"exit_code": 0} + ) + ) + ], + ), + custom_metadata={"codex_event_type": "item_completed"}, + ) + streaming = _text_event("Final ans", author=author, partial=True) + answer = _text_event("Final answer.", author=author) + turn_complete = Event( + invocation_id="inv-1", + author=author, + turn_complete=True, + partial=True, + custom_metadata={"codex_event_type": "turn_complete", "turn_id": "turn-1"}, + ) + return [reasoning, tool_call, tool_response, streaming, answer, turn_complete] @pytest.mark.asyncio @pytest.mark.parametrize("runtime", ["codex", "piagent"]) async def test_non_adk_runtime_saves_output_key(monkeypatch, runtime: str) -> None: - final_event = _text_event("runtime result", author="worker") + turn = _codex_shaped_turn() class FakeRuntime: async def run_async(self, agent, ctx): - yield final_event + for event in turn: + yield event monkeypatch.setattr("veadk.runtime.get_runtime", lambda name: FakeRuntime()) agent = Agent( @@ -143,8 +191,55 @@ async def run_async(self, agent, ctx): events = [event async for event in agent._run_async_impl(SimpleNamespace())] - assert events == [final_event] - assert final_event.actions.state_delta == {"result": "runtime result"} + assert events == turn + written = [ + (index, event.actions.state_delta) + for index, event in enumerate(events) + if event.actions.state_delta + ] + assert written == [(4, {"result": "Final answer."})], written + + +@pytest.mark.asyncio +async def test_multiple_final_text_events_write_output_key_once(monkeypatch) -> None: + """Several complete assistant messages in one turn: the last one wins. + + ``is_final_response()`` is True for every non-partial tool-free text event, + so a harness that streams N completed messages runs the save N times. The + surviving value must be the agent's last message, never a concatenation of + its intermediate thinking. + """ + turn = [ + _text_event("Let me check that.", author="worker"), + _text_event("Almost there.", author="worker"), + _text_event("Final answer.", author="worker"), + ] + + class FakeRuntime: + async def run_async(self, agent, ctx): + for event in turn: + yield event + + monkeypatch.setattr("veadk.runtime.get_runtime", lambda name: FakeRuntime()) + agent = Agent( + name="worker", + runtime="codex", + model_api_key="test-key", + output_key="result", + ) + + events = [event async for event in agent._run_async_impl(SimpleNamespace())] + + assert [e.actions.state_delta["result"] for e in events] == [ + "Let me check that.", + "Almost there.", + "Final answer.", + ] + # Merged in session order, the last write is what a downstream node reads. + merged: dict = {} + for event in events: + merged.update(event.actions.state_delta) + assert merged == {"result": "Final answer."} @pytest.mark.asyncio diff --git a/tests/test_adk_compat_regression.py b/tests/test_adk_compat_regression.py index 10d82d080..fe694cd55 100644 --- a/tests/test_adk_compat_regression.py +++ b/tests/test_adk_compat_regression.py @@ -535,34 +535,32 @@ def _runner_env(monkeypatch, tmp_path): def _make_fake_runner(_runner_env): - """Build a Runner with a never-called FakeLlm so we can inspect plumbing.""" - from typing import AsyncGenerator + """Build a Runner over a scripted model so we can inspect the plumbing. - from google.adk.models.base_llm import BaseLlm - from google.adk.models.llm_response import LlmResponse - from google.genai import types + Uses the differential suite's :class:`ScriptedBackend` rather than a local + one-line ``BaseLlm``: the local one could not script multiple rounds or + record what it was asked for, and two divergent ADK model fakes in one repo + is how the two runtimes drifted apart in the first place. + """ + import sys + from pathlib import Path + + harness = Path(__file__).resolve().parent / "runtime" / "differential" + sys.path.insert(0, str(harness)) + from scripted_backend import Round, ScriptedBackend from veadk import Agent, Runner from veadk.memory.short_term_memory import ShortTermMemory - class FakeLlm(BaseLlm): - async def generate_content_async( - self, llm_request, stream=False - ) -> AsyncGenerator[LlmResponse, None]: - yield LlmResponse( - content=types.Content( - role="model", - parts=[types.Part(text="fake reply")], - ) - ) - + backend = ScriptedBackend([Round(text="fake reply")], arm="adk") agent = Agent( name="fake_agent", description="fake", instruction="be brief", - model=FakeLlm(model="fake"), + model=backend.as_base_llm(model="fake"), ) runner = Runner(agent=agent, short_term_memory=ShortTermMemory(backend="local")) + runner._scripted_backend = backend return runner diff --git a/tests/test_agent.py b/tests/test_agent.py index 5371d7d7c..f04cc393d 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -15,6 +15,7 @@ import os from unittest.mock import Mock, PropertyMock, patch +import pytest from google.adk.agents.llm_agent import LlmAgent from google.adk.models.lite_llm import LiteLlm from google.adk.tools import load_memory @@ -34,6 +35,18 @@ def test_agent(): + # `KnowledgeBase(backend="local")` and `LongTermMemory(backend="local")` + # import their llama-index-backed backend classes lazily, at construction + # time, so this test body -- not the module import -- is where the missing + # extra bites. The rest of this module runs without it. + pytest.importorskip( + "llama_index.core", + reason=( + "the local KnowledgeBase/LongTermMemory backends need llama-index: " + 'pip install "veadk-python[extensions]"' + ), + ) + os.environ["MODEL_EMBEDDING_API_KEY"] = "mocked_api_key" knowledgebase = KnowledgeBase(index="test_index", backend="local") diff --git a/tests/test_knowledgebase.py b/tests/test_knowledgebase.py index 971e1ba66..56319430e 100644 --- a/tests/test_knowledgebase.py +++ b/tests/test_knowledgebase.py @@ -16,8 +16,19 @@ import pytest -from veadk.knowledgebase import KnowledgeBase -from veadk.knowledgebase.backends.in_memory_backend import InMemoryKnowledgeBackend +# `veadk.knowledgebase.backends.in_memory_backend` imports `llama_index.core` at +# module scope, so without the `extensions` extra this module cannot even be +# collected. Skip the module on that one import rather than guarding the whole +# file, so an unrelated ImportError still surfaces as an error. +pytest.importorskip( + "llama_index.core", + reason='KnowledgeBase needs llama-index: pip install "veadk-python[extensions]"', +) + +from veadk.knowledgebase import KnowledgeBase # noqa: E402 +from veadk.knowledgebase.backends.in_memory_backend import ( # noqa: E402 + InMemoryKnowledgeBackend, +) @pytest.mark.asyncio diff --git a/tests/test_long_term_memory.py b/tests/test_long_term_memory.py index feaec3719..70228767b 100644 --- a/tests/test_long_term_memory.py +++ b/tests/test_long_term_memory.py @@ -86,6 +86,17 @@ def test_memory_auto_save_policy_is_exported(): @pytest.mark.asyncio async def test_long_term_memory(): + # Only the `local` backend needs llama-index, and it is imported lazily + # when the backend class is resolved; every other test in this module + # injects its own backend and runs without the extra. + pytest.importorskip( + "llama_index.core", + reason=( + "the local KnowledgeBase/LongTermMemory backends need llama-index: " + 'pip install "veadk-python[extensions]"' + ), + ) + os.environ["MODEL_EMBEDDING_API_KEY"] = "mocked_api_key" long_term_memory = LongTermMemory(backend="local") diff --git a/tests/test_runner.py b/tests/test_runner.py index b28712251..c45903cba 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -14,6 +14,7 @@ import os +import pytest from google.genai import types from veadk.agent import Agent @@ -67,6 +68,16 @@ def _test_convert_messages(runner): def test_runner(): """Test Runner class initialization and core properties""" + # `LongTermMemory(backend="local")` below resolves its backend class + # lazily, and the local one imports `llama_index.core`. + pytest.importorskip( + "llama_index.core", + reason=( + "the local KnowledgeBase/LongTermMemory backends need llama-index: " + 'pip install "veadk-python[extensions]"' + ), + ) + os.environ["MODEL_EMBEDDING_API_KEY"] = "mocked_api_key" short_term_memory = ShortTermMemory() diff --git a/tests/test_short_term_memory.py b/tests/test_short_term_memory.py index 75364f504..4952f4ad8 100644 --- a/tests/test_short_term_memory.py +++ b/tests/test_short_term_memory.py @@ -21,8 +21,7 @@ from veadk.utils.misc import formatted_timestamp -def test_short_term_memory(): - # local +def test_short_term_memory_local(): memory = ShortTermMemory(backend="local") asyncio.run( memory.session_service.create_session( @@ -36,7 +35,25 @@ def test_short_term_memory(): ) assert session is not None - # sqlite + +def test_short_term_memory_sqlite(): + # Every non-`local` backend goes through ADK's DatabaseSessionService, whose + # async engine runs the sync DBAPI inside a greenlet. SQLAlchemy declares + # greenlet only for a fixed platform_machine list (aarch64/x86_64/amd64/ + # win32) that excludes macOS arm64, so a plain `pip install veadk-python` + # leaves it missing there; `sqlalchemy[asyncio]` requires it unconditionally + # and is what veadk-python[extensions] pulls in transitively (via + # llama-index-core). On Linux/Windows CI runners it is always present, so + # this never skips there. + pytest.importorskip( + "greenlet", + reason=( + "SQLAlchemy's async engine needs greenlet: " + 'pip install "sqlalchemy[asyncio]" ' + '(also pulled in by "veadk-python[extensions]")' + ), + ) + memory = ShortTermMemory( backend="sqlite", local_database_path=f"/tmp/tmp_for_test_{formatted_timestamp()}.db", diff --git a/veadk/agent.py b/veadk/agent.py index 41106ebb9..e6c54199b 100644 --- a/veadk/agent.py +++ b/veadk/agent.py @@ -187,6 +187,16 @@ class Agent(LlmAgent): enable_skills_checklist: bool = False _skills_with_checklist: Dict[str, Any] = {} + _veadk_explicit_fields: Optional[frozenset] = None + """Field names the caller actually passed to ``Agent(...)``. + + Snapshotted at the top of :meth:`model_post_init`, before this class starts + assigning ``model``, ``model_extra_config`` and ``run_processor`` itself. + ``model_fields_set`` is unusable for "did the user set this?" afterwards: + those assignments add themselves to it, and ``BaseAgent.clone()`` + re-assigns every list field on the copy. Consumed by + :func:`veadk.runtime.compat.explicit_fields`.""" + runtime: Literal["adk", "codex", "piagent"] = "adk" """Agent runtime backend. ``"adk"`` (default) uses Google ADK's built-in LLM flow. ``"codex"`` delegates the inner agent loop to the OpenAI Codex SDK. @@ -218,6 +228,11 @@ class Agent(LlmAgent): `veadk.tunnel.mount_tunnel`/`mount_tunnel_if_enabled`.""" def model_post_init(self, __context: Any) -> None: + # Snapshot before anything below assigns fields on ``self``: pydantic + # adds every assignment to ``model_fields_set``, so this is the only + # point at which "the caller set this" is still knowable. + self._veadk_explicit_fields = frozenset(self.model_fields_set) + super().model_post_init(None) # for sub_agents init # Toolsets that create sub-agents at runtime need ADK to select its @@ -459,8 +474,31 @@ def model_post_init(self, __context: Any) -> None: f"Agent: {self.model_dump(include={'id', 'name', 'model_name', 'model_api_base', 'tools', 'skills'})}" ) + if self.runtime != "adk": + # Fail at ``Agent(...)`` rather than at the first turn. This is a + # convenience, not the authoritative gate: ``BaseAgent.clone()`` + # uses ``model_copy(update=...)``, which runs neither validators nor + # ``model_post_init``, and ``spawn_harness_agent`` flips ``runtime`` + # through exactly that path. ``_run_async_impl`` re-checks. + from veadk.runtime.compat import check_agent_runtime_support + + check_agent_runtime_support(self, self.runtime) + def update_model(self, model_name: str): + """Point the agent at a different model. + + Both model sources are updated. ``self.model`` is what ADK's LLM flow + calls, while the external runtimes resolve the model from + ``self.model_name`` (``CodexRuntime._resolve_model``, + ``PiAgentModelConfig.from_agent``). Updating only ``self.model`` made + per-request model overrides a no-op under ``runtime="codex"`` / + ``"piagent"``. + + Args: + model_name (str): The new model name, without a provider prefix. + """ logger.info(f"Updating model to {model_name}") + self.model_name = model_name self.model = self.model.model_copy( update={"model": f"{self.model_provider}/{model_name}"} ) @@ -748,12 +786,49 @@ async def _run_async_impl( return from veadk.runtime import get_runtime + from veadk.runtime.compat import check_agent_runtime_support from veadk.runtime.output_state import maybe_save_output_to_state + # Authoritative support-matrix gate. It has to live here rather than in + # a validator or in the runtime: ``BaseAgent.clone()`` bypasses + # validators and ``model_post_init`` (and ``spawn_harness_agent`` flips + # ``runtime`` through it), while the runtimes themselves are also driven + # directly with bare ``LlmAgent``/duck-typed agents by their own tests. + check_agent_runtime_support( + self, + self.runtime, + run_config=getattr(ctx, "run_config", None), + ) + async for event in get_runtime(self.runtime).run_async(self, ctx): maybe_save_output_to_state(self, event) yield event + async def _run_live_impl( + self, ctx: "InvocationContext" + ) -> AsyncGenerator["Event", None]: + """Run the live/bidi loop, which only the ``adk`` runtime implements. + + ``LlmAgent._run_live_impl`` goes straight to ``self._llm_flow.run_live``, + which VeADK does not override. Without this guard a + ``runtime="codex"`` agent reached through ADK's ``/run_live`` endpoint + would silently run the full ADK flow — a different model loop, a + different tool set and no Codex sandbox — instead of its configured + runtime. + + Raises: + NotImplementedError: If the agent uses a non-``adk`` runtime. + """ + if self.runtime != "adk": + raise NotImplementedError( + f"Agent(runtime={self.runtime!r}) has no live/bidi " + "implementation; run_live would silently fall back to the ADK " + "flow and use a different model loop and tool set. Use " + "runner.run_async, or set runtime='adk' for live sessions." + ) + async for event in super()._run_live_impl(ctx): + yield event + if not is_adk_gte("2.0.0"): # On google-adk 1.x, BaseAgent has no `run` method, so override here # to nudge users toward `runner.run_async`. On google-adk 2.x, diff --git a/veadk/cli/cli_harness.py b/veadk/cli/cli_harness.py index 7c207a68e..537717579 100644 --- a/veadk/cli/cli_harness.py +++ b/veadk/cli/cli_harness.py @@ -233,7 +233,7 @@ done; \\ test -d src/veadk RUN uv pip install --system --index-url https://mirrors.aliyun.com/pypi/simple/ \\ - "./src[harness]" fastapi "uvicorn[standard]" + "./src[harness,codex]" fastapi "uvicorn[standard]" EXPOSE 8000 CMD ["python", "-m", "uvicorn", "veadk.cloud.harness_app.app:app", "--host", "0.0.0.0", "--port", "8000"] """ diff --git a/veadk/cloud/harness_app/Dockerfile b/veadk/cloud/harness_app/Dockerfile index 6aa549fc9..a127a5f90 100644 --- a/veadk/cloud/harness_app/Dockerfile +++ b/veadk/cloud/harness_app/Dockerfile @@ -24,8 +24,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends git && \ apt-get purge -y git && apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* -# To run with the "codex" runtime (RUNTIME=codex), also install: -# pip3 install --no-cache-dir openai-codex +# The harness advertises `runtime: codex` in harness.yaml and also honours a +# per-request runtime override, so the image must be able to serve it. Installed +# unconditionally: a missing extra would otherwise surface as an ImportError on +# the first request of an already-deployed runtime. +ARG INSTALL_CODEX=1 +RUN if [ "$INSTALL_CODEX" = "1" ]; then \ + pip3 install --no-cache-dir "openai-codex==0.1.0b3" "openai-codex-cli-bin==0.137.0a4"; \ + fi COPY agent.py app.py ./ diff --git a/veadk/integrations/ve_faas/template/{{cookiecutter.local_dir_name}}/src/requirements.txt b/veadk/integrations/ve_faas/template/{{cookiecutter.local_dir_name}}/src/requirements.txt index 3b50313a5..6445dcb98 100644 --- a/veadk/integrations/ve_faas/template/{{cookiecutter.local_dir_name}}/src/requirements.txt +++ b/veadk/integrations/ve_faas/template/{{cookiecutter.local_dir_name}}/src/requirements.txt @@ -1,3 +1,7 @@ veadk-python=={{ cookiecutter.veadk_version }} fastapi uvicorn[standard] +# If any agent in this app uses `Agent(runtime="codex")`, uncomment the line +# below. Without it the deployment builds and starts fine but every invocation +# fails with an ImportError on the first request. +# veadk-python[codex]=={{ cookiecutter.veadk_version }} diff --git a/veadk/runtime/codex/__init__.py b/veadk/runtime/codex/__init__.py index 6da035c92..959ff58b7 100644 --- a/veadk/runtime/codex/__init__.py +++ b/veadk/runtime/codex/__init__.py @@ -16,6 +16,11 @@ The implementation is imported lazily so configuration, translation, and tool bridge helpers remain usable when the optional ``openai-codex`` SDK is absent. + +:func:`current_workspace` is deliberately *not* lazy: an ADK tool imports it to +find the directory the sandbox is working in, and that tool module has to stay +importable in processes where the Codex SDK is not installed (the same tool is +routinely run by other runtimes, and by unit tests). """ from __future__ import annotations @@ -23,8 +28,9 @@ from typing import Any from veadk.runtime.codex.config import CodexRuntimeConfig +from veadk.runtime.codex.workspace import current_workspace -__all__ = ["CodexRuntime", "CodexRuntimeConfig"] +__all__ = ["CodexRuntime", "CodexRuntimeConfig", "current_workspace"] def __getattr__(name: str) -> Any: diff --git a/veadk/runtime/codex/config.py b/veadk/runtime/codex/config.py index 8937aede6..a79afccce 100644 --- a/veadk/runtime/codex/config.py +++ b/veadk/runtime/codex/config.py @@ -21,7 +21,11 @@ from pathlib import Path from typing import Literal -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator + +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) _KEY_ENV = "VEADK_CODEX_API_KEY" _SENSITIVE_ENV_MARKERS = ( @@ -44,16 +48,56 @@ class CodexRuntimeConfig(BaseModel): write only inside a session-isolated workspace, escalated operations are denied, and network access is disabled. Applications that need broader access must opt in explicitly. + + Security note on ``approval_mode``: ``"auto_review"`` is **not** a review + gate. The Codex SDK's built-in approval handler answers every + ``requestApproval`` notification with ``accept``, and ``AsyncCodex`` exposes + no hook to replace it, so ``"auto_review"`` auto-approves every sandbox + escalation and file change without consulting a human or ADK. Only + ``"deny_all"`` (the default) actually keeps Codex inside the sandbox. + + Security note on ``network_access``: it is written to the + ``[sandbox_workspace_write]`` table of Codex's ``config.toml`` and is read + only by the ``workspace-write`` sandbox. The ``read-only`` and + ``danger-full-access`` sandboxes ignore it entirely, so it cannot restrict + (or grant) network access outside ``sandbox="workspace_write"``. """ - approval_mode: Literal["deny_all", "auto_review"] = "deny_all" + approval_mode: Literal["deny_all", "auto_review"] = Field( + default="deny_all", + description=( + "'deny_all' refuses every escalation Codex requests. 'auto_review' " + "AUTO-APPROVES them: the SDK's default approval handler accepts " + "every command-execution and file-change approval request, and no " + "human or ADK confirmation is consulted. Treat 'auto_review' as " + "full auto-approval, not as a review step." + ), + ) sandbox: Literal["read_only", "workspace_write", "full_access"] = "workspace_write" - network_access: bool = False + network_access: bool = Field( + default=False, + description=( + "Allow network access from the sandbox. Only honoured by " + "sandbox='workspace_write'; 'read_only' and 'full_access' ignore it." + ), + ) workspace_root: str | None = None reuse_workspace: bool = False reasoning_effort: Literal["minimal", "low", "medium", "high", "xhigh"] = "medium" personality: Literal["none", "friendly", "pragmatic"] = "pragmatic" - max_tool_iterations: int = Field(default=8, ge=1, le=64) + max_tool_iterations: int = Field( + default=32, + ge=1, + le=256, + description=( + "ADK tool round-trips the shim may run for the whole Codex turn. " + "This budget is per turn, not per backend request: Codex issues one " + "request per native tool round, so a per-request counter allowed " + "rounds x budget executions. The default is higher than the old " + "per-request value so that turns which use an ADK tool after " + "several native tool rounds are not cut short." + ), + ) tool_timeout_seconds: float | None = Field(default=120.0, gt=0) @field_validator("workspace_root") @@ -63,6 +107,40 @@ def _normalize_workspace_root(cls, value: str | None) -> str | None: return None return str(Path(value).expanduser().resolve()) + @model_validator(mode="after") + def _check_sandbox_network_consistency(self) -> "CodexRuntimeConfig": + """Reject sandbox/network combinations that misstate the isolation. + + ``network_access`` only reaches Codex through the + ``[sandbox_workspace_write]`` table, which ``danger-full-access`` and + ``read-only`` ignore. Silently accepting those combinations lets a + config read as "no network" while granting full network, so the + dangerous direction raises and the harmless one warns. + + Returns: + CodexRuntimeConfig: The validated config. + + Raises: + ValueError: If ``sandbox="full_access"`` is combined with + ``network_access=False``. + """ + if self.sandbox == "full_access" and not self.network_access: + raise ValueError( + "CodexRuntimeConfig(sandbox='full_access') ignores " + "network_access, so network_access=False gives no network " + "isolation. Set sandbox='workspace_write' to actually block " + "network access, or set network_access=True to acknowledge " + "the risk." + ) + if self.sandbox == "read_only" and self.network_access: + logger.warning( + "CodexRuntimeConfig(sandbox='read_only') ignores " + "network_access=True, which only applies to the " + "workspace_write sandbox. Set sandbox='workspace_write' if " + "the agent needs network access." + ) + return self + @classmethod def from_agent(cls, agent: object) -> "CodexRuntimeConfig": """Resolve config from the Agent field with narrow environment fallbacks.""" diff --git a/veadk/runtime/codex/proxy.py b/veadk/runtime/codex/proxy.py index aa9e7f95f..697b6b642 100644 --- a/veadk/runtime/codex/proxy.py +++ b/veadk/runtime/codex/proxy.py @@ -21,26 +21,42 @@ Responses requests and forwards them through :func:`litellm.aresponses` — whose completion-transformation bridge converts Responses ⇄ chat-completions — to the backend. Codex is then pointed at the local server. + +The agent's ADK tools are deliberately *not* exposed to Codex: they are +advertised to the backend as plain ``function`` tools and executed by the shim +itself in a bounded internal loop (see :class:`TurnToolState`). Codex never sees +them, so the shim — not Codex — owns their conversation history. """ from __future__ import annotations import asyncio +import atexit +import contextlib +import hashlib import json import os import secrets +import threading import time -from dataclasses import dataclass -from typing import Any, AsyncIterator +import weakref +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, AsyncIterator, Callable import litellm import uvicorn from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, StreamingResponse -from litellm.exceptions import APIError +from litellm import exceptions as litellm_exceptions from veadk.utils.logger import get_logger +try: # OpenTelemetry is optional; the shim must import without it. + from opentelemetry import context as otel_context_api +except Exception: # pragma: no cover - depends on the install extras + otel_context_api = None # type: ignore[assignment] + logger = get_logger(__name__) _TRANSFERRED_STATUS = "transferred" @@ -75,6 +91,13 @@ def _shim_num_retries() -> int: the eval client's read timeout (default 300s) fired before any recovery. Retrying lets litellm apply its built-in exponential backoff. Env-tunable via ``CODEX_SHIM_NUM_RETRIES`` (default 2). + + These are retries of a *failed* attempt, so they multiply HTTP requests but + not model calls: one charge against ``max_llm_calls`` can cost up to + ``1 + CODEX_SHIM_NUM_RETRIES`` requests, doubled again if + :func:`_call_backend_tolerating_reasoning` has to repair the request. See + ``ShimTurnContext.on_model_call`` for why the budget counts calls rather + than attempts. """ try: return max(0, int(os.getenv("CODEX_SHIM_NUM_RETRIES", "2"))) @@ -93,6 +116,70 @@ def _shim_timeout() -> float: return 0.0 +def _shim_start_timeout() -> float: + """Deadline (seconds) for the local server to bind its ephemeral port. + + Without a deadline ``start()`` polls ``server.started`` forever, so an + environment where uvicorn cannot bind (no loopback, port exhaustion, a + restricted network namespace) hangs the whole invocation instead of failing + it. Env-tunable via ``CODEX_SHIM_START_TIMEOUT`` (default 10s). + """ + try: + return max(0.5, float(os.getenv("CODEX_SHIM_START_TIMEOUT", "10"))) + except ValueError: + return 10.0 + + +def _shim_cache_max() -> int: + """Maximum number of cached shims (one local server + port each). + + The process-wide cache is keyed by backend + credential, so in a + multi-tenant server a per-tenant key would otherwise allocate an unbounded + number of servers and ports. Env-tunable via ``CODEX_SHIM_CACHE_MAX``. + """ + try: + return max(1, int(os.getenv("CODEX_SHIM_CACHE_MAX", "8"))) + except ValueError: + return 8 + + +def _shim_reserve_seconds() -> float: + """Floor on how long a shim handed out by :func:`get_shim` counts as busy. + + ``get_shim`` returns well before the caller can ``register_turn``: the Codex + runtime first prepares a workspace, reaps stale ones (up to 16 ``rmtree``\ s + in a worker thread), prepares a ``CODEX_HOME``, syncs skills, and + builds/resumes its toolsets (which connects MCP servers). Across that whole + window the shim has no registered turn, so a concurrent ``get_shim`` for a + different backend could evict it — stopping its server and releasing its + port — and the turn would then register on a corpse and point Codex at a + dead URL for its entire duration. + + This value alone cannot close that window, because the window has no bound + the shim can know: MCP connect timeouts and workspace reaping are the + caller's business, and any constant is a guess that a slow setup outlives. + What actually holds the reservation open is the :class:`ShimLease` + ``get_shim`` returns — the shim keeps only a weak reference to it, so the + reservation lives exactly as long as the caller's own reference does, for + any setup duration. This deadline is the *floor* underneath that, for a + caller that keeps only the URL and drops the lease (:func:`get_shim_url`) + or that never held one at all. + + Neither half is a caller-released counter, and that is deliberate: a release + call would have to survive every exit path of an async generator (including + a consumer abandoning it mid-setup), and one missed release would pin a shim + in the cache forever. Dropping the last reference to a lease is not a call + that can be missed — the interpreter always makes it, on every exit path — + and :meth:`register_turn` consumes the reservation atomically with inserting + the turn, so the normal path never waits for the floor to expire. + Env-tunable via ``CODEX_SHIM_RESERVE_SECONDS``; ``0`` disables reservations. + """ + try: + return max(0.0, float(os.getenv("CODEX_SHIM_RESERVE_SECONDS", "60"))) + except ValueError: + return 60.0 + + def _bearer_token(request: Request) -> str: authorization = request.headers.get("authorization", "") scheme, _, token = authorization.partition(" ") @@ -106,19 +193,357 @@ def _openai_error(*, status_code: int, error_type: str, message: str) -> JSONRes ) -# Cap on shim-internal tool round-trips per turn — bounds runaway loops while -# allowing several tool calls per turn. +# Fallback cap on shim-internal tool round-trips per Codex *turn*, used only +# when `register_turn` is called without `max_tool_iterations`. The product +# default is `CodexRuntimeConfig.max_tool_iterations` (32), which the Codex +# runtime always passes explicitly; this value exists so a direct +# `register_turn` caller (tests, embedders) still gets a bounded loop. _AGENT_TOOL_MAX_ITERS = 8 +# Hard ceiling on retained per-turn tool transcript items (function_call + +# function_call_output pairs replayed to the backend). Purely a memory guard: +# the iteration budget already bounds the number of rounds. +_TURN_TRANSCRIPT_MAX_ITEMS = 256 + +# Grace period for a shim's uvicorn server to drain on stop(). +_SHIM_STOP_TIMEOUT = 5.0 + +# SSE `response.failed` error code used for every terminal shim failure. +# +# Codex classifies a `response.failed` frame by `response.error.code` +# (`codex-rs/codex-api/src/sse/responses.rs`, the `"response.failed"` arm): +# `context_length_exceeded`, `insufficient_quota`, `usage_not_included`, +# `cyber_policy` and `invalid_prompt` map to *fatal* `ApiError` variants, and +# **everything else falls through to `ApiError::Retryable`** — which becomes +# `CodexErr::Stream`, whose `is_retryable()` is true, so Codex re-sends the +# request `stream_max_retries` times with backoff. A descriptive code such as +# `tool_iteration_limit` therefore buys N pointless retries, each one a fresh +# backend call. `invalid_prompt` is the only fatal code that *keeps* our message +# (it maps to `CodexErr::InvalidRequest(message)`); the others discard it. On a +# Codex build without that mapping the behaviour is simply today's retry-then- +# fail, so this is a strict improvement or a no-op, never a regression. +_FATAL_STREAM_ERROR_CODE = "invalid_prompt" + + +class TurnToolState: + """Mutable, concurrency-safe per-turn state for the shim's tool loop. + + Two things must outlive a single HTTP request but stay scoped to one Codex + turn: + + * the **iteration budget** — Codex sends one request per native tool + round, so a per-request counter let the shim run ``max_tool_iterations`` + backend round-trips *per request* (N x 8 per turn); + * the **tool transcript** — the ``function_call``/``function_call_output`` + pairs the shim executed itself. Codex never sees those items (they are + not streamed to it), and Codex rebuilds the whole ``input`` array from + its own thread on every request, so without replaying them the model + would see a conversation in which it never called the tool and would + re-issue the call (re-running its side effects). + + The shim is shared process-wide and serves concurrent turns; a plain + ``threading.Lock`` is used rather than an ``asyncio`` primitive so the state + is safe from any thread/event loop (``register_turn`` runs on the caller's + loop, the handler on the server's). + """ + + __slots__ = ( + "_lock", + "_transcript", + "_iterations", + "_dropped", + "_error", + "_anchor_texts", + "_marker_delivered", + ) + + def __init__(self) -> None: + self._lock = threading.Lock() + self._transcript: list[dict[str, Any]] = [] + self._iterations = 0 + self._dropped = 0 + self._error: BaseException | None = None + # Turn identity, latched on the first request this turn serves. `None` + # means "no request seen yet"; see `identify_request`. + self._anchor_texts: frozenset[str] | None = None + self._marker_delivered = False + + @property + def iterations(self) -> int: + """Tool round-trips consumed so far in this turn.""" + with self._lock: + return self._iterations + + @property + def error(self) -> BaseException | None: + """Exception that aborted the turn from inside the shim, if any. + + The shim runs on the server's task, so an exception raised here (for + example ADK's ``LlmCallsLimitExceededError`` from the per-model-call + budget) cannot propagate to the caller of ``run_async``. It is recorded + instead, and the runtime re-raises it once the turn ends so the normal + ADK handling in ``veadk.runner`` still applies. + """ + with self._lock: + return self._error + + def record_error(self, error: BaseException) -> None: + """Remember the first shim-side exception that aborted this turn.""" + with self._lock: + if self._error is None: + self._error = error + + @property + def transcript(self) -> list[dict[str, Any]]: + """Copy of the executed tool items recorded for this turn.""" + with self._lock: + return [dict(item) for item in self._transcript] + + def consume_iteration(self, budget: int) -> bool: + """Reserve one tool round-trip; ``False`` when the turn budget is gone.""" + with self._lock: + if self._iterations >= budget: + return False + self._iterations += 1 + return True + + def identify_request( + self, marker: str, user_texts: list[str], *, tools_advertised: bool + ) -> bool: + """Decide whether this inbound request is the agent's own turn. + + Codex reuses one provider block — and therefore one bearer token — for + work that is *not* the agent turn: auto/manual compaction + (``codex-rs/core/src/compact.rs::run_compact_task_inner_impl``, which + re-sends the whole history plus a summarization instruction with an + empty ``tools`` list) and code review (``session/review.rs``, which + clones the parent turn's provider into a *fresh* delegate thread). If + the shim advertised the agent's ADK tools to those passes, or replayed + the turn's tool transcript into them, the summarizer could emit a + ``function_call`` and the shim would execute a real tool a second time. + + Identification is positive, never by exclusion: + + * **Marker path** (production). The runtime embeds this turn's + ``turn_marker`` in the Codex prompt, so the marker travels in the + turn's own user message. Compaction *preserves* that message — it + re-sends the full history — so the marker alone is not enough; the + request must additionally look like a *sampling* pass rather than a + summarization one. Two independent signals say so, and either + suffices: the marker is in the **last** user message (compaction + appends its instruction after it, so this is false there), or the + request advertises a non-empty ``tools`` list (compaction always + sends ``[]``; this is the arm that keeps ADK tools working for the + rest of a turn *after* a mid-turn compaction has appended its summary + as a trailing user message). + * **Anchor path** (fallback). If the marker never arrives on the first + request — an SDK or Codex version that reshapes the prompt — the shim + degrades to matching the first request's user-message texts instead of + failing the turn's tools closed for good. The first request under a + freshly registered token is by construction the turn's opening + sampling request: nothing exists yet to compact and no review thread + can have been spawned. Later requests are then held to the same + two-signal test as the marker path, because a compaction pass re-sends + the whole history and so always carries the anchor texts. + + Residual risk, stated rather than hidden: both paths lean on compaction + sending an empty ``tools`` list. A Codex version that advertises tools + on a summarization pass would satisfy the ``tools_advertised`` arm and + be treated as the agent turn. That arm cannot simply be dropped — it is + what keeps ADK tools working for the rest of a turn after a mid-turn + compaction appends its summary as a trailing user message. + + Returns: + bool: ``True`` when tool injection, transcript replay and the + shim's tool loop may run for this request. + """ + with self._lock: + first = self._anchor_texts is None + if first: + self._anchor_texts = frozenset(user_texts) + self._marker_delivered = bool(marker) and any( + marker in text for text in user_texts + ) + anchors = self._anchor_texts or frozenset() + marker_delivered = self._marker_delivered + if marker and marker_delivered: + if not any(marker in text for text in user_texts): + return False + return tools_advertised or (bool(user_texts) and marker in user_texts[-1]) + if first: + return True + # Anchored on the *last* user message only. Matching any remembered + # text would admit a compaction pass, which re-sends the whole history + # and so always carries the turn's opening message. The marker path's + # `tools_advertised` arm is deliberately not mirrored here: without a + # marker it would also admit a review pass, which runs in a fresh + # delegate thread but does advertise tools. Losing ADK tools for the + # rest of a turn after a mid-turn compaction is a degradation; running + # a real tool inside a summarization or review pass is a wrong answer + # with side effects, so this fallback fails closed. + return bool(user_texts) and user_texts[-1] in anchors + + def marker_was_delivered(self) -> bool: + """Whether the turn marker was seen on this turn's first request.""" + with self._lock: + return self._marker_delivered + + def record(self, items: list[dict[str, Any]]) -> None: + """Append executed ``function_call``/``function_call_output`` items.""" + if not items: + return + with self._lock: + self._transcript.extend(dict(item) for item in items) + overflow = len(self._transcript) - _TURN_TRANSCRIPT_MAX_ITEMS + if overflow <= 0: + return + # Trim to whole pairs. The cap is even and items are appended two at + # a time, so an even prefix is pair-aligned today — but that is an + # accident of the caller, not a property of this method. A lone + # `function_call_output` at the head would reach the backend as a + # `tool` message with no preceding `assistant(tool_calls)`, which + # Ark rejects outright, so the boundary is advanced explicitly until + # the head is not an orphaned result. + while overflow < len(self._transcript) and ( + self._transcript[overflow].get("type") == "function_call_output" + ): + overflow += 1 + del self._transcript[:overflow] + self._dropped += overflow + + def replay_items(self, seen_call_ids: set[str]) -> list[dict[str, Any]]: + """Items to re-append to a fresh request's ``input``. + + Anything whose ``call_id`` is already present in the inbound request is + skipped, so the pairs can never be duplicated (both items of a pair + share a ``call_id``, so a pair is always kept or dropped whole). + """ + with self._lock: + return [ + dict(item) + for item in self._transcript + if item.get("call_id") not in seen_call_ids + ] + @dataclass(frozen=True) class ShimTurnContext: - """Immutable tool routing data for one Codex invocation.""" + """Tool routing data for one Codex invocation. + + The dataclass itself stays frozen (routing data is fixed for the turn); + mutable per-turn state lives in :attr:`state`. + """ specs: tuple[dict[str, Any], ...] executors: dict[str, Any] max_tool_iterations: int invocation_id: str = "" + # Opaque per-turn string the runtime embeds in the Codex prompt so the shim + # can tell this turn's own sampling requests from Codex-internal passes + # (compaction, review) that arrive on the same bearer token. See + # `TurnToolState.identify_request`. + turn_marker: str = "" + # Per-turn backend attribution/caching config (Ark prompt caching, + # veadk-source/version headers, ...). Attached per turn rather than to the + # shim because one shim instance is shared by every turn on a backend. + extra_headers: dict[str, str] = field(default_factory=dict) + extra_body: dict[str, Any] = field(default_factory=dict) + # OTel context captured on the invocation's own task, re-attached around + # tool execution so ADK `execute_tool` spans keep their real parent. + otel_context: Any = None + # Charged once per *model call* — one iteration of the shim's tool loop — + # not once per HTTP request. ADK's `max_llm_calls` budget lives on the + # InvocationContext, and the real model calls happen here rather than in the + # runtime, so the runtime hands the counter down. Raising from it aborts the + # turn (see `TurnToolState.error`). + # + # Calls, not attempts, is the deliberate reading and it matches the `adk` + # arm: ADK charges `increment_llm_call_count` once per `BaseLlmFlow` call + # while litellm's own `num_retries` re-attempts underneath it, so counting + # HTTP requests here would make the same agent hit `max_llm_calls` at + # different points on the two runtimes and break the differential parity the + # suite asserts. It is also the reading that means something: a retried + # attempt produced no response and (on a completion-billed backend such as + # Ark) no charge, so counting it would spend a budget the user is not + # paying. The fan-out is bounded and worth stating -- one charge admits at + # most `1 + CODEX_SHIM_NUM_RETRIES` requests, and up to twice that if + # `_call_backend_tolerating_reasoning` retries the request without Codex's + # replayed reasoning items (that repair is the *same* logical call: the + # first attempt was rejected outright, so it never produced a response). + on_model_call: Callable[[], None] | None = None + state: TurnToolState = field(default_factory=TurnToolState) + + +@dataclass +class _Reservation: + """One outstanding ``get_shim`` -> ``register_turn`` handoff. + + Two independent things keep it alive, and either suffices: + + * ``holder`` — a weak reference to the :class:`ShimLease` handed to the + caller. While that reference is alive the handoff is still in progress, + however long the caller's setup takes. + * ``deadline`` — the ``CODEX_SHIM_RESERVE_SECONDS`` floor, for a caller that + kept only the shim's URL and dropped the lease. + """ + + deadline: float + holder: "weakref.ReferenceType[ShimLease]" + + def is_live(self, now: float) -> bool: + return self.holder() is not None or self.deadline > now + + +class ShimLease: + """A shim borrowed from the process-wide cache, plus its reservation. + + :func:`get_shim` returns one of these rather than the bare + :class:`ResponsesShim`. Every attribute access delegates to the shim, so a + caller uses it exactly as it used the shim; what the lease adds is + *liveness*. The shim holds only a weak reference back, so the reservation + that keeps it un-evictable lasts precisely as long as the caller's own + reference to this object — the entire ``get_shim`` -> ``register_turn`` + setup, whatever that costs on the day, and not one moment past the frame + that owns it. + + That is deliberately not a release call: there is nothing to call, so there + is nothing to miss on an exception, a ``GeneratorExit``, or a consumer that + abandons the runtime's async generator mid-setup — the interpreter drops the + reference on every one of those paths. The only way to pin a shim is to keep + a lease alive on purpose, which is an explicit strong reference and no + different from keeping the shim itself. + + :meth:`register_turn` is overridden so the turn consumes *this* caller's + reservation and no one else's; see :meth:`ResponsesShim.register_turn`. + """ + + __slots__ = ("_shim", "_reservation_id", "__weakref__") + + def __init__(self, shim: ResponsesShim) -> None: + self._shim = shim + self._reservation_id: int | None = None + + @property + def shim(self) -> ResponsesShim: + """The leased shim itself, for a caller that needs it unwrapped.""" + return self._shim + + def register_turn(self, *args: Any, **kwargs: Any) -> str: + """:meth:`ResponsesShim.register_turn`, consuming this reservation.""" + kwargs.setdefault("reservation", self) + return self._shim.register_turn(*args, **kwargs) + + def __getattr__(self, name: str) -> Any: + # Both slots are assigned in `__init__`, so normal lookup finds them and + # this is never reached for them on a built object. Guarding anyway: + # on a half-built one (unpickling, a subclass that skips `__init__`) + # delegating `_shim` would recurse until the stack ran out. + if name in ("_shim", "_reservation_id"): + raise AttributeError(name) + return getattr(self._shim, name) + + def __repr__(self) -> str: + return f"ShimLease(url={getattr(self._shim, 'url', None)!r})" class ResponsesShim: @@ -140,12 +565,78 @@ def __init__(self, api_base: str, api_key: str) -> None: self.url: str | None = None self._server: uvicorn.Server | None = None self._task: asyncio.Task[Any] | None = None + self._loop: asyncio.AbstractEventLoop | None = None + # Serializes cold start: `start()` polls until the socket is bound, and + # without the lock two concurrent invocations (serverless cold start + # with a burst) each build and serve a uvicorn.Server, leaking the + # loser's socket forever. + self._start_lock: asyncio.Lock | None = None + self._start_lock_loop: asyncio.AbstractEventLoop | None = None # Invocation-scoped registry. The opaque token is supplied to the Codex # subprocess as its provider API key and arrives as a Bearer token, so # concurrent turns can never overwrite one another's tools/context. self._turns: dict[str, ShimTurnContext] = {} + # Outstanding `get_shim` handoffs whose turn has not been registered + # yet, keyed by an id that is never reused so one caller can only ever + # consume its own; see `_shim_reserve_seconds` and `ShimLease`. + self._reservations: dict[int, _Reservation] = {} + self._reservation_seq = 0 + self._turns_lock = threading.Lock() self._app = self._build_app() + @property + def busy(self) -> bool: + """True while this shim has a registered turn or a live reservation. + + The reservation half is what makes the ``get_shim`` -> ``register_turn`` + handoff safe: without it the shim reports idle across the caller's whole + setup window and the LRU can evict (and stop) it out from under a turn + that is about to register. + """ + with self._turns_lock: + if self._turns: + return True + self._prune_reservations_locked() + return bool(self._reservations) + + def _prune_reservations_locked(self) -> None: + """Drop reservations whose lease is gone *and* whose floor has passed.""" + now = time.monotonic() + for reservation_id in [ + key + for key, reservation in self._reservations.items() + if not reservation.is_live(now) + ]: + del self._reservations[reservation_id] + + def reserve(self) -> ShimLease: + """Borrow this shim, keeping it un-evictable while the caller sets up. + + Returns: + ShimLease: A handle that delegates every attribute to this shim. + Hold it for the whole ``get_shim`` -> ``register_turn`` handoff: the + handle *is* the reservation, and the shim tracks it weakly, so + letting go of it releases the reservation (no sooner than the + ``CODEX_SHIM_RESERVE_SECONDS`` floor, which covers a caller that + keeps only the URL). Registering a turn through the handle consumes + it immediately. + """ + lease = ShimLease(self) + window = _shim_reserve_seconds() + if window <= 0: + # Reservations disabled by config. The lease is still returned so + # callers need no branch of their own; it simply protects nothing. + return lease + with self._turns_lock: + self._prune_reservations_locked() + self._reservation_seq += 1 + reservation_id = self._reservation_seq + self._reservations[reservation_id] = _Reservation( + deadline=time.monotonic() + window, holder=weakref.ref(lease) + ) + lease._reservation_id = reservation_id + return lease + def register_turn( self, specs: list[dict[str, Any]], @@ -153,63 +644,229 @@ def register_turn( *, max_tool_iterations: int = _AGENT_TOOL_MAX_ITERS, invocation_id: str = "", + model_extra_config: dict[str, Any] | None = None, + on_model_call: Callable[[], None] | None = None, + reservation: ShimLease | None = None, ) -> str: - """Register immutable routing state and return its opaque bearer token.""" + """Register one invocation's routing state; returns its bearer token. + + Args: + specs: ADK tool specs advertised to the backend as ``function`` tools. + executors: ``name -> async (args, call_id) -> str`` tool executors. + max_tool_iterations: Tool round-trip budget for the whole turn. + invocation_id: ADK invocation id, for logs. + model_extra_config: The agent's ``model_extra_config`` + (``extra_headers``/``extra_body``), forwarded to the backend on + every call of this turn. Defaults to no extra config. + on_model_call: Charged once per backend model call, before the call + is made -- per *call*, not per HTTP attempt; see + ``ShimTurnContext.on_model_call``. Used to enforce ADK's + ``RunConfig.max_llm_calls``, whose counter lives on the + invocation the runtime owns. Raising from it aborts the turn; + the exception is recorded on the turn state and surfaced to + Codex as a ``429``. + reservation: The lease :func:`get_shim` handed this caller, whose + reservation the new turn takes over. Callers that never reserved + (tests, embedders) leave it unset and consume nothing. + """ token = secrets.token_urlsafe(32) - self._turns[token] = ShimTurnContext( + headers, body = _split_model_extra_config(model_extra_config) + context = ShimTurnContext( specs=tuple(specs or ()), executors=dict(executors or {}), max_tool_iterations=max(1, max_tool_iterations), invocation_id=invocation_id, + # Generated here, not derived from `token`: the marker is embedded + # in the model-visible prompt, and the token is the shim's bearer + # credential. + turn_marker=f"veadk-turn-{secrets.token_hex(12)}", + extra_headers=headers, + extra_body=body, + # Captured here because register_turn runs on the invocation's own + # task; the server task's context was snapshotted at shim start. + otel_context=( + otel_context_api.get_current() if otel_context_api is not None else None + ), + on_model_call=on_model_call, ) + with self._turns_lock: + # Consume *this caller's* reservation -- and only it -- in the same + # critical section that publishes the turn, so the shim is + # continuously `busy` across the handoff and can never be evicted + # between the two. Popping an arbitrary reservation instead would + # make a caller that never reserved (registering directly, which is + # supported) cancel the protection of whichever *other* caller is in + # setup right now, re-opening for that turn exactly the window this + # exists to close. + self._prune_reservations_locked() + reservation_id = getattr(reservation, "_reservation_id", None) + if reservation_id is not None: + self._reservations.pop(reservation_id, None) + self._turns[token] = context logger.debug( - "codex_shim_turn_registered invocation_id=%s tool_count=%d", + "codex_shim_turn_registered invocation_id=%s tool_count=%d " + "extra_header_count=%d extra_body_count=%d", invocation_id, len(executors or {}), + len(headers), + len(body), ) return token def unregister_turn(self, token: str) -> None: """Remove one invocation's routing state.""" - context = self._turns.pop(token, None) + with self._turns_lock: + context = self._turns.pop(token, None) if context is not None: logger.debug( - "codex_shim_turn_unregistered invocation_id=%s", + "codex_shim_turn_unregistered invocation_id=%s tool_iterations=%d", context.invocation_id, + context.state.iterations, ) + def _turn(self, token: str) -> ShimTurnContext | None: + with self._turns_lock: + return self._turns.get(token) + + def turn_marker(self, token: str) -> str: + """Opaque marker the caller must embed in this turn's Codex prompt. + + The shim uses it to tell the agent's own sampling requests apart from + Codex-internal passes that arrive on the same bearer token (compaction, + review). Returns ``""`` for an unknown token, in which case the shim + falls back to matching the turn's first request; see + :meth:`TurnToolState.identify_request`. + """ + context = self._turn(token) + return context.turn_marker if context is not None else "" + + def turn_error(self, token: str) -> BaseException | None: + """Exception raised inside the shim that aborted this turn, if any. + + Read it *before* :meth:`unregister_turn`, which drops the turn state. + The shim serves requests on the server's task, so an exception from + ``on_model_call`` cannot reach the runtime by propagation; the runtime + re-raises whatever is returned here once the Codex turn ends. + """ + context = self._turn(token) + return context.state.error if context is not None else None + def _build_app(self) -> FastAPI: app = FastAPI() + # Registered on `app` by the decorator; the name is never referenced + # again, and must stay as-is because FastAPI derives the route's + # OpenAPI operation id from it. @app.post("/v1/responses") async def responses(request: Request) -> Any: token = _bearer_token(request) - turn_context = self._turns.get(token) + turn_context = self._turn(token) if turn_context is None: return _openai_error( status_code=401, error_type="authentication_error", message="Unknown or expired Codex invocation token.", ) - body = await request.json() + try: + body = await request.json() + except Exception: # noqa: BLE001 - malformed client payload + return _openai_error( + status_code=400, + error_type="invalid_request_error", + message="Request body is not valid JSON.", + ) + if not isinstance(body, dict) or not body.get("model"): + return _openai_error( + status_code=400, + error_type="invalid_request_error", + message="Request body must be an object with a `model`.", + ) model = body["model"] stream = bool(body.get("stream", False)) + def _fail( + *, + status_code: int, + error_type: str, + message: str, + template: dict[str, Any] | None = None, + ) -> Any: + """Terminal shim error, in the shape the client can parse. + + A streaming client cannot consume an HTTP error body: it is + parsing an event stream, so the status code is all it sees and a + 4xx/5xx is classified purely by Codex's transport rules. It gets + ``response.failed`` instead, which carries a real message — and + :data:`_FATAL_STREAM_ERROR_CODE`, so Codex treats the failure as + terminal rather than replaying the request `stream_max_retries` + times. ``error_type`` stays the caller's own name; it is what the + non-streaming JSON body reports. + """ + if stream: + return StreamingResponse( + _synth_failed_sse( + template or {"model": model}, + code=_FATAL_STREAM_ERROR_CODE, + message=f"{error_type}: {message}", + ), + media_type="text/event-stream", + ) + return _openai_error( + status_code=status_code, + error_type=error_type, + message=message, + ) + call_kwargs: dict[str, Any] = { key: body[key] for key in _PASSTHROUGH_KEYS if key in body } - # Advertise the agent's ADK tools to the backend as plain `function` - # tools; the shim executes them itself (see the tool loop below). - # Codex's own non-`function` tools are dropped, since Ark rejects - # their schema (e.g. the hosted `web_search`'s `external_web_access`); - # this leaves Codex's web search disabled on a chat backend. - agent_executors = turn_context.executors - if isinstance(call_kwargs.get("tools"), list): - kept = [t for t in call_kwargs["tools"] if t.get("type") == "function"] - have = {t.get("name") for t in kept} - kept.extend(t for t in turn_context.specs if t.get("name") not in have) + # Is this the agent's own turn, or a Codex-internal pass (auto + # compaction / review) on the same bearer token? Only the former may + # be handed the agent's tools, the turn's tool transcript, or the + # shim's tool loop; see `TurnToolState.identify_request`. + inbound_tools = call_kwargs.get("tools") + is_agent_turn = turn_context.state.identify_request( + turn_context.turn_marker, + _user_message_texts(call_kwargs.get("input")), + tools_advertised=isinstance(inbound_tools, list) + and bool(inbound_tools), + ) + # Identification is done; the marker is shim-internal routing data + # and must not reach the model. + _strip_turn_marker(call_kwargs.get("input"), turn_context.turn_marker) + if not is_agent_turn: + logger.info( + "codex_shim_internal_pass_forwarded invocation_id=%s " + "detail=request is not this turn's sampling pass; ADK tools " + "and tool history are withheld.", + turn_context.invocation_id, + ) + elif turn_context.turn_marker and not ( + turn_context.state.marker_was_delivered() + ): + logger.warning( + "codex_shim_turn_marker_missing invocation_id=%s " + "detail=the turn marker never reached the model request; " + "falling back to first-request matching to identify this " + "turn's own passes.", + turn_context.invocation_id, + ) + # Drop Codex's own non-`function` tools unconditionally: Ark rejects + # their schema (e.g. the hosted `web_search`'s `external_web_access`), + # and that is a backend-compatibility concern rather than part of + # tool advertisement. Only *adding* the agent's ADK tools — which + # the shim executes itself, see the tool loop below — is gated on + # this being the agent turn. + agent_executors = turn_context.executors if is_agent_turn else {} + if isinstance(inbound_tools, list): + kept = [t for t in inbound_tools if t.get("type") == "function"] + if is_agent_turn: + have = {t.get("name") for t in kept} + kept.extend( + t for t in turn_context.specs if t.get("name") not in have + ) call_kwargs["tools"] = kept - elif turn_context.specs: + elif is_agent_turn and turn_context.specs: call_kwargs["tools"] = list(turn_context.specs) # On multi-step turns Codex replays prior assistant messages in # `input` without a `status` field, but Ark's Responses API @@ -225,6 +882,28 @@ async def responses(request: Request) -> Any: and "status" not in item ): item["status"] = "completed" + + # Replay this turn's shim-executed tool history. Codex rebuilds + # `input` from its own thread on every request and never saw the + # ADK function_call/function_call_output pairs (they are not + # streamed to it, precisely so Codex does not try to dispatch tools + # it does not own), so without this the model would see a + # conversation in which it never called the tool and would re-issue + # the call — re-running its side effects. Pairs are appended at the + # tail (never spliced mid-array) so the chat bridge always sees an + # assistant(tool_calls) message immediately followed by its tool + # result, and are skipped when their call_id is already present. + conversation = call_kwargs.get("input") + if is_agent_turn and isinstance(conversation, list): + replay = turn_context.state.replay_items(_call_ids(conversation)) + if replay: + conversation.extend(replay) + logger.debug( + "codex_shim_tool_history_replayed invocation_id=%s items=%d", + turn_context.invocation_id, + len(replay), + ) + call_kwargs.update( model=f"openai/{model}", api_base=self.api_base, @@ -234,6 +913,13 @@ async def responses(request: Request) -> Any: num_retries=_shim_num_retries(), stream=False, ) + # Ark prompt caching / attribution headers, mirroring what the + # `adk` runtime path sends (veadk/agent.py merges + # DEFAULT_MODEL_EXTRA_CONFIG into model_extra_config). + if turn_context.extra_headers: + call_kwargs["extra_headers"] = dict(turn_context.extra_headers) + if turn_context.extra_body: + call_kwargs["extra_body"] = dict(turn_context.extra_body) timeout = _shim_timeout() if timeout: call_kwargs["timeout"] = timeout @@ -256,10 +942,49 @@ async def responses(request: Request) -> Any: # the loop is disabled and the path is unchanged for tool-less runs. exec_names = set(agent_executors) max_iters = turn_context.max_tool_iterations if agent_executors else 0 - iters = 0 + # Codex reads a request's token cost off `response.completed`, and + # only the final backend response is returned to it. Every + # intermediate call the tool loop makes is just as billable, so the + # blocks are summed and the total replaces the last one's usage + # before either return path. + usage_acc: dict[str, int] = {} + resp: dict[str, Any] = {} while True: - result = await litellm.aresponses(**call_kwargs) + # Charge ADK's per-invocation model-call budget here: this is + # where the calls actually happen — including on a Codex-internal + # pass, which is just as billable and just as capable of looping, + # so leaving it uncharged would put a hole in the very budget + # `max_llm_calls` exists to enforce. + # + # Raising aborts the turn. It must be reported the same way every + # other terminal shim failure is: a streaming client is parsing + # an event stream and cannot read an HTTP error body, so a bare + # 429 is classified by transport rules alone and the message is + # lost. (The abort is *not* dangerous to retry — `on_model_call` + # raises before the backend call and before any tool runs, so a + # retry is a pure no-op 429; the cost is latency and log noise, + # and the fatal `response.failed` code avoids paying it.) + if turn_context.on_model_call is not None: + try: + turn_context.on_model_call() + except Exception as e: # noqa: BLE001 - relayed to the runtime + turn_context.state.record_error(e) + logger.warning( + "codex_shim_turn_aborted invocation_id=%s error_type=%s", + turn_context.invocation_id, + type(e).__name__, + ) + return _fail( + status_code=429, + error_type="llm_calls_limit", + message=str(e), + template=_with_total_usage( + resp or {"model": model}, usage_acc + ), + ) + result = await _call_backend_tolerating_reasoning(call_kwargs) resp = _to_dict(result) + _accumulate_usage(usage_acc, resp.get("usage")) if max_iters <= 0: break conv = call_kwargs.get("input") @@ -274,24 +999,28 @@ async def responses(request: Request) -> Any: if not calls: break - if iters >= max_iters: + # Budget is per turn, not per request: Codex issues a fresh + # request after every native tool call, so a per-request counter + # allowed max_iters round-trips each time. + if not turn_context.state.consume_iteration(max_iters): logger.warning( "codex_tool_iteration_limit invocation_id=%s limit=%d", turn_context.invocation_id, max_iters, ) - return _openai_error( + return _fail( status_code=409, error_type="tool_iteration_limit", message=( "Codex tool iteration budget exhausted " - f"after {max_iters} round(s)." + f"after {max_iters} round(s) this turn." ), + template=_with_total_usage(resp, usage_acc), ) async def _execute( fc: dict[str, Any], - ) -> tuple[dict[str, Any], str, bool]: + ) -> tuple[dict[str, Any], str]: cid = fc.get("call_id") or fc.get("id") try: args = json.loads(fc.get("arguments") or "{}") @@ -304,7 +1033,6 @@ async def _execute( "status": "failed", } ), - False, ) if not isinstance(args, dict): return ( @@ -315,17 +1043,58 @@ async def _execute( "status": "failed", } ), - False, ) - out = await agent_executors[fc["name"]](args, str(cid)) - return fc, out, _is_transfer_output(out) - - executed = await asyncio.gather(*(_execute(fc) for fc in calls)) + # Re-attach the invocation's OTel context: this coroutine + # runs in a task descended from the uvicorn server task, + # whose contextvars were snapshotted when the shim first + # started, so ADK's `execute_tool` span would otherwise be + # an orphan root with a foreign trace_id. + with _otel_scope(turn_context.otel_context): + out = await agent_executors[fc["name"]](args, str(cid)) + return fc, out + + # `return_exceptions=True` so a sibling is never left running + # detached: a bare `gather` re-raises the first failure and + # abandons the rest, which then keep executing real tools and + # pushing ADK events into the runtime's queue long after this + # handler returned — possibly after the done sentinel. Every + # coroutine is awaited to completion here, then the first + # failure (if any) is reported. + settled = await asyncio.gather( + *(_execute(fc) for fc in calls), return_exceptions=True + ) + cancelled = next( + (r for r in settled if isinstance(r, asyncio.CancelledError)), None + ) + if cancelled is not None: + raise cancelled + failure = next( + (r for r in settled if isinstance(r, BaseException)), None + ) + if failure is not None: + logger.error( + "codex_tool_execution_failed invocation_id=%s error_type=%s", + turn_context.invocation_id, + type(failure).__name__, + ) + return _fail( + status_code=500, + error_type="tool_execution_error", + message=( + f"Tool execution failed: {type(failure).__name__}: " + f"{failure}" + ), + template=_with_total_usage(resp, usage_acc), + ) + executed: list[tuple[dict[str, Any], str]] = [ + r for r in settled if not isinstance(r, BaseException) + ] + pairs: list[dict[str, Any]] = [] transferred = False - for fc, out, did_transfer in executed: + for fc, out in executed: cid = fc.get("call_id") or fc.get("id") - transferred = transferred or did_transfer - conv.append( + transferred = transferred or _is_transfer_output(out) + pairs.append( { "type": "function_call", "call_id": cid, @@ -335,80 +1104,480 @@ async def _execute( "status": "completed", } ) - conv.append( + pairs.append( { "type": "function_call_output", "call_id": cid, "output": out, } ) + conv.extend(pairs) + # Remember them for the *next* request of this same turn. + turn_context.state.record(pairs) if transferred: resp = _completed_transfer_response(resp) break - iters += 1 + resp = _with_total_usage(resp, usage_acc) if stream: return StreamingResponse( _synth_sse(resp), media_type="text/event-stream" ) return JSONResponse(resp) - @app.exception_handler(APIError) - async def _on_api_error(_request: Request, exc: APIError) -> JSONResponse: - status = getattr(exc, "status_code", 500) or 500 + # Starlette calls exception handlers as `handler(request, exc)`, so the + # first parameter must exist even though this handler ignores it. + async def _on_backend_error(request: Request, exc: Exception) -> JSONResponse: + status = _error_status(exc) + detail = self._redact(str(getattr(exc, "message", None) or exc)) + # Record it against the turn so the runtime re-raises after the + # stream ends. Codex treats a rejected request as the end of its + # turn and returns whatever it had, so without this the caller sees + # `status=completed`, a half-finished workspace and a plausible + # summary -- a silently wrong answer. A 4xx here is terminal: it + # arrives only after litellm's own retries are exhausted. + turn_context = self._turn(_bearer_token(request)) + if turn_context is not None: + turn_context.state.record_error(exc) logger.warning( - "codex_backend_api_error status_code=%s error_type=%s", + "codex_backend_api_error status_code=%s error_type=%s detail=%s", status, type(exc).__name__, + detail, ) return JSONResponse( status_code=status, - content={ - "error": { - "type": _error_type(status), - "message": getattr(exc, "message", str(exc)), - } - }, + content={"error": {"type": _error_type(status), "message": detail}}, ) + # Starlette resolves handlers by walking `type(exc).__mro__`, and none + # of the errors litellm actually raises inherit `litellm.APIError` + # (RateLimitError/AuthenticationError/BadRequestError derive from + # openai.APIStatusError, Timeout from openai.APITimeoutError). Register + # every root so 400/401/408/429/5xx are mapped instead of escaping to + # ServerErrorMiddleware as a plain-text 500 that Codex then retries. + for exc_type in _backend_error_types(): + app.add_exception_handler(exc_type, _on_backend_error) # type: ignore[arg-type] + return app + def _redact(self, message: str) -> str: + """Strip the backend credential and cap the size of an error message.""" + if self.api_key and self.api_key in message: + message = message.replace(self.api_key, "***") + return message if len(message) <= 2000 else message[:2000] + "... [truncated]" + + def usable_on(self, loop: asyncio.AbstractEventLoop) -> bool: + """Whether this cached shim is still serving on ``loop``. + + A shim started on another (or a closed) event loop — e.g. a serverless + worker that runs each invocation under its own ``asyncio.run`` — has an + unreachable server, so the cache must drop it rather than hand back a + dead URL. + """ + if self._loop is None: + return True # never started + if self._loop is not loop or loop.is_closed(): + return False + # A shim whose server task is still running is usable even before + # `url` is assigned: a concurrent caller that arrives mid-bind must + # wait on `start()`, not discard (and force_close) the live server. + return self._task is None or not self._task.done() + async def start(self) -> str: """Start the server on an ephemeral local port and return its URL.""" if self.url: return self.url - # The shim app has no startup/shutdown hooks, so disable the lifespan - # protocol; otherwise its task lingers and logs a CancelledError - # traceback when the event loop is torn down at process exit. - config = uvicorn.Config( - self._app, - host="127.0.0.1", - port=0, - log_level="warning", - lifespan="off", - ) - server = uvicorn.Server(config) - server.install_signal_handlers = lambda: None # type: ignore[method-assign] - self._server = server - self._task = asyncio.create_task(server.serve()) - - while not server.started: - await asyncio.sleep(0.02) - - port = server.servers[0].sockets[0].getsockname()[1] - self.url = f"http://127.0.0.1:{port}" - logger.info("codex_shim_started listen_url=%s", self.url) - return self.url - - async def stop(self) -> None: - """Stop the server and await its task.""" - if self._server is not None: - self._server.should_exit = True - if self._task is not None: - await self._task + loop = asyncio.get_running_loop() + # Lazily bound so the lock always belongs to the running loop (no await + # between the check and the assignment, so this is atomic). + if self._start_lock is None or self._start_lock_loop is not loop: + self._start_lock = asyncio.Lock() + self._start_lock_loop = loop + + async with self._start_lock: + if self.url: + return self.url + + # The shim app has no startup/shutdown hooks, so disable the + # lifespan protocol; otherwise its task lingers and logs a + # CancelledError traceback when the event loop is torn down at + # process exit. + config = uvicorn.Config( + self._app, + host="127.0.0.1", + port=0, + log_level="warning", + lifespan="off", + ) + server = uvicorn.Server(config) + server.install_signal_handlers = lambda: None # type: ignore[method-assign] + task = asyncio.create_task(server.serve()) + self._server = server + self._task = task + self._loop = loop + + deadline = time.monotonic() + _shim_start_timeout() + while not server.started: + if task.done(): + # Retrieve the exception so it is never "never retrieved", + # and fail the invocation instead of spinning forever. + exc = None if task.cancelled() else task.exception() + self._reset() + raise RuntimeError( + "Codex Responses shim server exited before binding" + ) from exc + if time.monotonic() >= deadline: + server.should_exit = True + task.cancel() + with contextlib.suppress(BaseException): + await task + self._reset() + raise TimeoutError( + "Codex Responses shim server did not bind within " + f"{_shim_start_timeout():.1f}s" + ) + await asyncio.sleep(0.02) + + try: + port = server.servers[0].sockets[0].getsockname()[1] + except (IndexError, AttributeError) as e: + server.should_exit = True + task.cancel() + with contextlib.suppress(BaseException): + await task + self._reset() + raise RuntimeError( + "Codex Responses shim server reported no bound socket" + ) from e + + self.url = f"http://127.0.0.1:{port}" + logger.info("codex_shim_started listen_url=%s", self.url) + return self.url + + def _reset(self) -> None: + self._server = None + self._task = None + self._loop = None self.url = None + async def stop(self, *, timeout: float = _SHIM_STOP_TIMEOUT) -> None: + """Ask the server to drain, await its task, and release the port.""" + running = None + with contextlib.suppress(RuntimeError): + running = asyncio.get_running_loop() + if self._loop is not None and self._loop is not running: + # The server task belongs to another (possibly closed) loop, so it + # cannot be awaited here; drop it synchronously instead. + self.force_close() + return + server, task = self._server, self._task + self._reset() + with self._turns_lock: + self._turns.clear() + if server is not None: + server.should_exit = True + if task is None: + return + try: + await asyncio.wait_for(task, timeout) + except asyncio.TimeoutError: + logger.warning("codex_shim_stop_timeout timeout_seconds=%s", timeout) + # A drain that overran its grace period leaves uvicorn still holding + # the listening socket, and `_reset()` has already dropped the + # instance attributes, so nothing would ever close it again: the + # port stays bound for the life of the process. The local `server` + # and `task` were captured before the reset, so the sockets can + # still be closed and the task cancelled here. + for bound in getattr(server, "servers", None) or (): + with contextlib.suppress(Exception): + bound.close() + task.cancel() + with contextlib.suppress(BaseException): + await task + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - shutdown must not raise + logger.warning("codex_shim_stop_failed error_type=%s", type(e).__name__) + else: + logger.info("codex_shim_stopped") + + def force_close(self) -> None: + """Best-effort synchronous teardown (no running loop required).""" + server, task = self._server, self._task + self._reset() + with self._turns_lock: + self._turns.clear() + if server is not None: + server.should_exit = True + for bound in getattr(server, "servers", None) or (): + with contextlib.suppress(Exception): + bound.close() + if task is not None: + with contextlib.suppress(Exception): + if task.done(): + if not task.cancelled(): + task.exception() # mark as retrieved + else: + task.cancel() + + +def _looks_like_reasoning_rejection(exc: BaseException) -> bool: + """Whether a backend error is "this model does not accept reasoning items".""" + text = str(getattr(exc, "message", None) or exc).lower() + return "reasoning" in text and ( + "not supported" in text or "unsupported" in text or "invalid" in text + ) + + +async def _call_backend_tolerating_reasoning(call_kwargs: dict[str, Any]) -> Any: + """Call the backend, retrying once without Codex's replayed reasoning items. + + After its first tool round Codex replays its own ``reasoning`` items in the + request ``input``. Some backends refuse them per-model -- Ark answers + ``InvalidParameter: input[N].reasoning ... not supported for model`` for + ``doubao-seed-1-6``, while accepting them for other models -- and the + rejection lands mid-investigation, so the model family silently became + unusable with this runtime rather than merely degraded. + + Reasoning items are dropped only in response to that specific refusal, never + pre-emptively: for a backend that accepts them they carry the chain of + thought across tool rounds, and stripping them unconditionally would trade a + hard failure on a few models for quieter, worse answers on the rest. + """ + try: + return await litellm.aresponses(**call_kwargs) + except Exception as e: # noqa: BLE001 - re-raised unless it is this one case + conversation = call_kwargs.get("input") + if not _looks_like_reasoning_rejection(e) or not isinstance(conversation, list): + raise + kept = [ + item + for item in conversation + if not (isinstance(item, dict) and item.get("type") == "reasoning") + ] + if len(kept) == len(conversation): + raise + logger.info( + "codex_backend_reasoning_items_dropped removed=%d", + len(conversation) - len(kept), + ) + return await litellm.aresponses(**{**call_kwargs, "input": kept}) + + +#: ``extra_body`` keys that the Responses transport cannot carry, so they are +#: dropped rather than forwarded. Ark rejects prompt caching when the request +#: also has an ``instructions`` field ("caching is not supported for +#: instructions"), and Codex *always* sends ``instructions`` -- so forwarding +#: VeADK's default ``caching`` block 400s every single turn. ``expire_at`` only +#: qualifies the cache entry, so it goes with it. Everything else in +#: ``extra_body`` is forwarded untouched. +_BODY_KEYS_UNSUPPORTED_ON_RESPONSES = ("caching", "expire_at") + + +def _split_model_extra_config( + model_extra_config: dict[str, Any] | None, +) -> tuple[dict[str, str], dict[str, Any]]: + """Normalize an agent ``model_extra_config`` into header/body dicts. + + Headers are forwarded in full -- they carry VeADK's Ark attribution and + encryption defaults. Body keys the Responses transport cannot support are + filtered; see :data:`_BODY_KEYS_UNSUPPORTED_ON_RESPONSES`. + """ + config = model_extra_config if isinstance(model_extra_config, dict) else {} + raw_headers = config.get("extra_headers") + raw_body = config.get("extra_body") + # The `isinstance` check has to guard the *iteration*, not each item: a + # comprehension evaluates `.items()` before it filters, so a truthy + # non-dict `extra_headers` (a list of pairs, say) raised `AttributeError` + # inside `register_turn` and failed the turn outright. `extra_body` below is + # the shape to match. + headers = ( + {str(k): str(v) for k, v in raw_headers.items() if v is not None} + if isinstance(raw_headers, dict) + else {} + ) + body = dict(raw_body) if isinstance(raw_body, dict) else {} + dropped = [key for key in _BODY_KEYS_UNSUPPORTED_ON_RESPONSES if key in body] + for key in dropped: + body.pop(key, None) + if dropped: + logger.debug( + "codex_shim_extra_body_filtered keys=%s", ",".join(sorted(dropped)) + ) + return headers, body + + +def _strip_turn_marker(items: Any, marker: str) -> int: + """Remove the shim's turn marker from a request's user messages, in place. + + The marker only has to survive the round trip *through Codex* — it is how + the shim recognises this turn's own sampling passes. The model must never + see it: it is routing metadata, it would differ from what the ADK runtime + sends for the identical agent (the differential parity suite asserts the two + arms send the same prompt), and a model that echoes it would corrupt the + answer. Codex keeps the marker in its own history either way, so stripping + here costs nothing. + + Args: + items (Any): The request's ``input`` array; non-lists are ignored. + marker (str): The turn marker to remove. + + Returns: + int: How many message parts were rewritten. + """ + if not marker or not isinstance(items, list): + return 0 + tag = f"{marker}" + stripped = 0 + for item in items: + if not isinstance(item, dict) or item.get("role") != "user": + continue + content = item.get("content") + if isinstance(content, str): + if tag in content: + item["content"] = content.replace(tag, "").rstrip() + stripped += 1 + continue + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict): + continue + text = part.get("text") + if isinstance(text, str) and tag in text: + part["text"] = text.replace(tag, "").rstrip() + stripped += 1 + return stripped + + +def _user_message_texts(items: Any) -> list[str]: + """Text of every ``user``-role message in a request's ``input``, in order. + + Used to identify the turn (see :meth:`TurnToolState.identify_request`). + Codex sends message content as a list of parts, but a hand-built request may + use a bare string, so both are accepted; non-message items (``reasoning``, + ``function_call``, ``function_call_output``) and the ``developer``-role + initial-context bundle are ignored. + """ + texts: list[str] = [] + if not isinstance(items, list): + return texts + for item in items: + if not isinstance(item, dict) or item.get("role") != "user": + continue + if item.get("type") not in (None, "message"): + continue + content = item.get("content") + if isinstance(content, str): + texts.append(content) + continue + if not isinstance(content, list): + continue + texts.append( + "\n".join( + str(part.get("text") or "") + for part in content + if isinstance(part, dict) + ) + ) + return texts + + +def _call_ids(items: list[Any]) -> set[str]: + """Collect the call/item ids already present in a request's ``input``.""" + seen: set[str] = set() + for item in items: + if not isinstance(item, dict): + continue + for key in ("call_id", "id"): + value = item.get(key) + if isinstance(value, str) and value: + seen.add(value) + return seen + + +@contextlib.contextmanager +def _otel_scope(context: Any): + """Attach ``context`` for the duration of the block, if OTel is available.""" + # Bind the optional module to a local so `attach` and `detach` are paired + # against the same non-None reference. `otel_context_api` is None on an + # install without OpenTelemetry, and a guard on the module global cannot be + # carried across the `yield` - neither by a reader nor by a type checker. + api = otel_context_api + token = None + if api is not None and context is not None: + try: + token = api.attach(context) + except Exception: # noqa: BLE001 - tracing must never break a tool + token = None + try: + yield + finally: + if api is not None and token is not None: + with contextlib.suppress(Exception): + api.detach(token) + + +def _backend_error_types() -> tuple[type[Exception], ...]: + """Exception classes that must map to OpenAI-shaped error JSON. + + Verified against the installed litellm/openai: the MRO of the errors that + actually occur is e.g. + ``RateLimitError -> openai.RateLimitError -> openai.APIStatusError -> + openai.APIError -> openai.OpenAIError``, so ``litellm.exceptions.APIError`` + (a sibling branch) is never found by Starlette's MRO walk. + ``openai.OpenAIError`` is the single common ancestor; the explicit litellm + names keep the mapping working if ``openai`` is not importable, and + ``BudgetExceededError`` derives straight from ``Exception``. + """ + candidates: list[type[Exception]] = [] + try: + from openai import OpenAIError as _OpenAIError + + candidates.append(_OpenAIError) + except Exception: # noqa: BLE001 - openai is optional at import time + pass + for name in ( + "OpenAIError", + "APIError", + "APIConnectionError", + "Timeout", + "RateLimitError", + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "BadRequestError", + "UnprocessableEntityError", + "InternalServerError", + "ServiceUnavailableError", + "APIResponseValidationError", + "BudgetExceededError", + ): + candidate = getattr(litellm_exceptions, name, None) + if isinstance(candidate, type) and issubclass(candidate, Exception): + candidates.append(candidate) + unique: list[type[Exception]] = [] + for candidate in candidates: + if candidate not in unique: + unique.append(candidate) + return tuple(unique) + + +def _error_status(exc: Exception) -> int: + """Best-effort HTTP status for a backend exception. + + Codex retries 429/5xx and surfaces 4xx, so preserving the backend's status + keeps its retry policy correct. + """ + status = getattr(exc, "status_code", None) + try: + status = int(status) # type: ignore[arg-type] + except (TypeError, ValueError): + return 500 + return status if 400 <= status <= 599 else 500 + def _error_type(status: int) -> str: """Map an HTTP status code to an error ``type`` string.""" @@ -417,7 +1586,10 @@ def _error_type(status: int) -> str: 401: "authentication_error", 403: "permission_error", 404: "not_found_error", + 408: "timeout_error", + 422: "invalid_request_error", 429: "rate_limit_error", + 503: "overloaded_error", }.get(status, "api_error") @@ -459,6 +1631,63 @@ def _completed_transfer_response(resp: dict[str, Any]) -> dict[str, Any]: } +# Usage counters summed across a turn's backend calls. The nested detail fields +# are addressed by their parent so a caller need not know the wire shape. +_USAGE_TOP_KEYS = ("input_tokens", "output_tokens", "total_tokens") +_USAGE_DETAIL_KEYS = ( + ("input_tokens_details", "cached_tokens"), + ("output_tokens_details", "reasoning_tokens"), +) + + +def _accumulate_usage(acc: dict[str, int], usage: Any) -> None: + """Add one backend response's token usage into ``acc``. + + The shim's tool loop calls the backend once per round but returns only the + final response, so without this every intermediate call's cost is dropped: + a turn billed for N calls would report the N-th call alone. Missing or + malformed blocks are ignored rather than raised on — usage is accounting, + and it must never fail a turn. + """ + if not isinstance(usage, dict): + return + for key in _USAGE_TOP_KEYS: + value = usage.get(key) + if isinstance(value, (int, float)): + acc[key] = acc.get(key, 0) + int(value) + for parent, key in _USAGE_DETAIL_KEYS: + detail = usage.get(parent) + if not isinstance(detail, dict): + continue + value = detail.get(key) + if isinstance(value, (int, float)): + acc[f"{parent}.{key}"] = acc.get(f"{parent}.{key}", 0) + int(value) + + +def _with_total_usage(resp: dict[str, Any], acc: dict[str, int]) -> dict[str, Any]: + """Return ``resp`` with its ``usage`` replaced by the turn's running total. + + Summing ``input_tokens`` counts the re-sent context once per call, which is + what the backend actually bills for an agentic loop. + """ + if not acc: + return resp + usage: dict[str, Any] = dict(resp.get("usage") or {}) + for key in _USAGE_TOP_KEYS: + if key in acc: + usage[key] = acc[key] + if "total_tokens" not in acc: + usage["total_tokens"] = acc.get("input_tokens", 0) + acc.get("output_tokens", 0) + for parent, key in _USAGE_DETAIL_KEYS: + total = acc.get(f"{parent}.{key}") + if total is None: + continue + detail = dict(usage.get(parent) or {}) + detail[key] = total + usage[parent] = detail + return {**resp, "usage": usage} + + def _sse(event: dict[str, Any]) -> bytes: """Encode one Responses event dict as an SSE frame.""" return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() @@ -475,6 +1704,12 @@ async def _synth_sse(resp: dict[str, Any]) -> AsyncIterator[bytes]: ``reasoning`` and ``function_call`` items are emitted; the last is what drives Codex's agentic loop (a dropped tool call ends the turn at the preamble). The completed response is trimmed to match what was streamed. + + Note that the shim's own ADK ``function_call``/``function_call_output`` + pairs are deliberately *not* streamed here: a ``function_call`` item is + Codex's "execute this" signal, and Codex has no such tool registered, so it + would answer with an ``unsupported call`` output and poison the thread. The + pairs are replayed to the backend instead (see :class:`TurnToolState`). """ seq = 0 @@ -580,22 +1815,215 @@ def ev(payload: dict[str, Any]) -> bytes: yield ev({"type": "response.completed", "response": completed}) -# Reuse one shim per (api_base, api_key) for the lifetime of the process. -_SHIMS: dict[tuple[str, str], ResponsesShim] = {} +async def _synth_failed_sse( + template: dict[str, Any], *, code: str, message: str +) -> AsyncIterator[bytes]: + """Synthesize a terminal ``response.failed`` stream. + ``response.failed`` (with ``response.error.{code,message}``) is a + recognized Codex error path, so the failure is reported once with a real + message instead of an HTTP 500 that Codex would retry — which would re-run + every tool side effect the request already produced. + """ + seq = 0 -async def get_shim(api_base: str, api_key: str) -> ResponsesShim: - """Return a started shim for the given backend, creating it if needed.""" - key = (api_base, api_key) - shim = _SHIMS.get(key) - if shim is None: - shim = ResponsesShim(api_base=api_base, api_key=api_key) - _SHIMS[key] = shim - await shim.start() - return shim + def ev(payload: dict[str, Any]) -> bytes: + nonlocal seq + payload["sequence_number"] = seq + seq += 1 + return _sse(payload) + + base = { + key: value + for key, value in (template or {}).items() + if key not in ("output", "status", "error") + } + base.setdefault("id", "resp_veadk_shim_error") + base.setdefault("object", "response") + yield ev( + { + "type": "response.created", + "response": {**base, "status": "in_progress", "output": []}, + } + ) + yield ev( + { + "type": "response.failed", + "response": { + **base, + "status": "failed", + "output": [], + "error": {"code": code, "message": message}, + }, + } + ) + + +# Reuse one shim per (api_base, credential fingerprint) for the lifetime of the +# process. LRU-ordered and capped so a multi-tenant server cannot allocate an +# unbounded number of servers/ports, and keyed by a hash so raw API keys are not +# retained in a module global. +_SHIMS: "OrderedDict[tuple[str, str], ResponsesShim]" = OrderedDict() + +# Guards every read-modify-write of `_SHIMS` and `_RETIRED`. +# +# A `threading.Lock` rather than an asyncio primitive, and not merely "no awaits +# in this block": that argument only makes a block atomic against other +# coroutines *on one event loop*, and this cache is explicitly designed to be +# shared across them -- `ResponsesShim.usable_on` exists for "a serverless +# worker that runs each invocation under its own `asyncio.run`". Two such +# workers on two threads both missed the cache and both built a `ResponsesShim`; +# the second `_SHIMS[key] = shim` orphaned the first, which then bound a port in +# `start()` while being reachable from nothing -- not `_evict_idle_shims`, not +# `shutdown_shims`, not `_close_shims_at_exit` -- so the socket leaked for the +# life of the process. `TurnToolState` and the reservations were already +# thread-safe; the cache holding them was not. +# +# Held only across dict operations, never across an await: a coroutine that +# blocked here while holding it would deadlock its own loop. +_SHIMS_LOCK = threading.Lock() + +# Shims removed from `_SHIMS` while still busy. A shim whose event loop no +# longer matches the caller's must leave the cache, but force-closing one that +# another loop's turn is still using would kill that turn, and simply dropping +# the reference would leak its port exactly the way an orphan does. They are +# parked here instead -- still serving, still reachable -- and closed by the +# next sweep once they go idle, or by shutdown/atexit. +_RETIRED: "list[ResponsesShim]" = [] + + +def _credential_fingerprint(api_key: str) -> str: + if not api_key: + return "" + return hashlib.sha256(api_key.encode("utf-8")).hexdigest()[:32] + + +async def get_shim(api_base: str, api_key: str) -> ShimLease: + """Return a started shim for the given backend, creating it if needed. + + Returns: + ShimLease: The shim, wrapped in the reservation that keeps it out of the + LRU's reach. Every attribute delegates to the shim, so it is used the + same way — but **hold the lease** for as long as the shim is needed: + letting go of it is what tells the cache the handoff is over. See + :class:`ShimLease`. + """ + key = (api_base, _credential_fingerprint(api_key)) + loop = asyncio.get_running_loop() + stale: ResponsesShim | None = None + retired: ResponsesShim | None = None + with _SHIMS_LOCK: + shim = _SHIMS.get(key) + if shim is not None and not shim.usable_on(loop): + del _SHIMS[key] + # Busy means another loop's turn is being served on it right now, + # so it is closed later (`_RETIRED`) rather than under that turn. + if shim.busy: + _RETIRED.append(shim) + retired = shim + else: + stale = shim + shim = None + if shim is None: + shim = ResponsesShim(api_base=api_base, api_key=api_key) + _SHIMS[key] = shim + _SHIMS.move_to_end(key) + # Reserved inside the same critical section that publishes the shim, so + # it is `busy` from the instant it becomes reachable: an `await` follows + # immediately below, and until this point a concurrent `get_shim` could + # evict a brand-new shim before its owner had ever been protected. + lease = shim.reserve() + if stale is not None or retired is not None: + logger.warning( + "codex_shim_discarded reason=event_loop_changed_or_closed retired=%s", + retired is not None, + ) + if stale is not None: + stale.force_close() + + try: + await shim.start() + except BaseException: + with _SHIMS_LOCK: + if _SHIMS.get(key) is shim: + del _SHIMS[key] + shim.force_close() + raise + + await _evict_idle_shims() + return lease + + +async def _evict_idle_shims() -> None: + """Drop least-recently-used shims that have no registered turn.""" + with _SHIMS_LOCK: + idle_retired = [shim for shim in _RETIRED if not shim.busy] + for shim in idle_retired: + _RETIRED.remove(shim) + for shim in idle_retired: + await shim.stop() + + limit = _shim_cache_max() + while True: + with _SHIMS_LOCK: + if len(_SHIMS) <= limit: + return + # The `busy` test and the pop have to be one critical section: a + # shim that reads idle here must still be idle when it is removed, + # or a turn registering in between is stopped out from under. + victim_key = next((k for k, s in _SHIMS.items() if not s.busy), None) + if victim_key is None: + logger.warning( + "codex_shim_cache_over_limit size=%d limit=%d reason=all_busy", + len(_SHIMS), + limit, + ) + return + victim = _SHIMS.pop(victim_key) + size = len(_SHIMS) + logger.info("codex_shim_evicted cache_size=%d limit=%d", size, limit) + await victim.stop() + + +async def shutdown_shims() -> None: + """Stop every cached shim. Call on worker/app shutdown for a clean drain.""" + while True: + with _SHIMS_LOCK: + if _SHIMS: + _, shim = _SHIMS.popitem() + elif _RETIRED: + shim = _RETIRED.pop() + else: + return + await shim.stop() + + +# Registered with `atexit` by the decorator; nothing calls it by name, and the +# registration side effect is the entire point, so it must not be deleted. +@atexit.register +def _close_shims_at_exit() -> None: + """Last-resort teardown: release listening sockets at interpreter exit.""" + # Best-effort on the lock: at exit a daemon thread may still hold it, and + # blocking forever here would hang the interpreter on the way out. Releasing + # the sockets matters more than the critical section, so a failed acquire + # falls through and drains anyway. + acquired = _SHIMS_LOCK.acquire(timeout=1.0) + try: + while _SHIMS or _RETIRED: + with contextlib.suppress(Exception): + shim = _SHIMS.popitem()[1] if _SHIMS else _RETIRED.pop() + shim.force_close() + finally: + if acquired: + _SHIMS_LOCK.release() async def get_shim_url(api_base: str, api_key: str) -> str: - """Return a started shim URL for the given backend, creating it if needed.""" - shim = await get_shim(api_base, api_key) - return shim.url or "" + """Return a started shim URL for the given backend, creating it if needed. + + The lease is dropped on return, so the shim is protected only by the + ``CODEX_SHIM_RESERVE_SECONDS`` floor. Callers that need the shim for a whole + turn should use :func:`get_shim` and keep the lease. + """ + lease = await get_shim(api_base, api_key) + return lease.url or "" diff --git a/veadk/runtime/codex/runtime.py b/veadk/runtime/codex/runtime.py index c077fd59b..bcc9c5f24 100644 --- a/veadk/runtime/codex/runtime.py +++ b/veadk/runtime/codex/runtime.py @@ -36,13 +36,15 @@ import asyncio import atexit +import enum import hashlib import os import shutil import tempfile +import threading import time from pathlib import Path -from typing import TYPE_CHECKING, AsyncGenerator +from typing import TYPE_CHECKING, Any, AsyncGenerator from openai_codex import ( # type: ignore[import-not-found] ApprovalMode, @@ -57,7 +59,6 @@ from openai_codex.generated.v2_all import ( # type: ignore[import-not-found] Personality, ReasoningEffort, - TurnCompletedNotification, ) from veadk.runtime.base_runtime import BaseRuntime @@ -82,13 +83,19 @@ from veadk.runtime.codex.translate import ( build_input_attachments_from_llm_request, build_prompt_from_llm_request, + build_turn_usage_metadata, + is_codex_final_text_event, notification_to_events, ) +from veadk.runtime.codex.workspace import ( + bind_workspace, + bind_workspace_to_executors, +) from veadk.runtime.model_callbacks import ( + merge_turn_bookkeeping, + RuntimeLlmCall, build_runtime_llm_request, final_events_to_llm_response, - has_after_model_callbacks, - is_final_model_text_event, llm_response_to_event, run_after_model_callbacks, run_before_model_callbacks, @@ -102,6 +109,8 @@ from google.adk.agents.invocation_context import InvocationContext from google.adk.events.event import Event from google.adk.models.llm_request import LlmRequest + from google.adk.models.llm_response import LlmResponse + from opentelemetry.trace import Span from veadk.agent import Agent @@ -109,9 +118,109 @@ _PROVIDER_ID = "veadk" _KEY_ENV = "VEADK_CODEX_API_KEY" -_QUEUE_DONE = object() -_SESSION_WORKSPACE_ROOT = tempfile.mkdtemp(prefix="veadk-codex-workspaces-") -atexit.register(shutil.rmtree, _SESSION_WORKSPACE_ROOT, ignore_errors=True) + + +class _QueueSentinel(enum.Enum): + """Single-member enum used as the event queue's end-of-stream marker. + + An `enum` member rather than a bare `object()` so that `is _QUEUE_DONE` + narrows the queue's `Event | BaseException | _QueueSentinel` union: the + queue multiplexes three kinds of payload onto one channel. + """ + + DONE = enum.auto() + + +_QUEUE_DONE = _QueueSentinel.DONE +_WORKSPACE_ROOT_PREFIX = "veadk-codex-workspaces-" +# The process-owned root that holds every session workspace, created on first +# use by `_ensure_session_workspace_root`. `None` until then. +_session_workspace_root: str | None = None +_session_workspace_root_lock = threading.Lock() +# Session workspaces are shared by every invocation of the same session, so +# they must outlive a turn. They are instead reaped once idle for this long, +# which bounds disk growth inside a long-lived server process. +_WORKSPACE_IDLE_TTL_SECONDS = 6 * 3600 +# The reaper walks a directory and removes trees, both of which are blocking +# syscalls. It runs off the event loop (a worker thread) *and* no more often +# than this, so a server handling many concurrent turns does not re-scan the +# root once per invocation. +_WORKSPACE_REAP_INTERVAL_SECONDS = 600.0 +# Upper bound on trees removed in one pass, so a root that accumulated +# thousands of stale sessions is drained over several passes instead of +# occupying a worker thread for an unbounded time. +_WORKSPACE_REAP_MAX_PER_PASS = 16 +_last_workspace_reap_at = 0.0 +_workspace_reap_lock = threading.Lock() + + +def _ensure_session_workspace_root() -> str: + """Return the process-owned session-workspace root, creating it on demand. + + Created on first use rather than at import time. As a module-level + ``tempfile.mkdtemp`` it ran for anything that merely *imported* this module + — a CLI listing runtimes, a test collecting, a worker that never served a + turn — and left a ``veadk-codex-workspaces-*`` directory in ``$TMPDIR`` + that the ``atexit`` hook can only reclaim on a clean exit. A ``SIGKILL`` + (an OOM kill, a torn-down ``pytest -xdist`` worker, a container stop) + orphans it, which is why a smoke run found several roots predating it. + Nothing but a real invocation needs the directory now, so nothing else + creates one. + + Returns: + str: Absolute path to the (existing) root directory. + """ + global _session_workspace_root + with _session_workspace_root_lock: + if _session_workspace_root is None: + root = tempfile.mkdtemp(prefix=_WORKSPACE_ROOT_PREFIX) + # Registered alongside creation, not at import: a hook over a + # directory that was never created is pure noise, and + # `ignore_errors=True` would have hidden that it did nothing. + atexit.register(shutil.rmtree, root, ignore_errors=True) + _session_workspace_root = root + return _session_workspace_root + + +def __getattr__(name: str) -> Any: + """Keep ``_SESSION_WORKSPACE_ROOT`` readable as a module attribute. + + PEP 562 module hook. The root is created lazily now, so it can no longer be + a module-level constant, but reading it must keep working (the smoke test + reads it to assert a turn left exactly one workspace behind). Access + materializes the root, because a caller that wants the path is about to + look inside it. + """ + if name == "_SESSION_WORKSPACE_ROOT": + return _ensure_session_workspace_root() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +#: Appended to every turn's developer instructions, because two things Codex's +#: own (deliberately preserved) system prompt tells the model are not true +#: here: +#: +#: - ``apply_patch`` never reaches the backend. Codex offers it as a non- +#: ``function`` tool and the shim forwards only ``function`` tools, so the +#: model is told to use a tool it was never given, and spends a round +#: discovering that. +#: - ``request_user_input`` *is* advertised, but nothing can answer it: an ADK +#: invocation has no interactive channel, so calling it ends the turn with +#: the work undone. +#: +#: Kept to the two facts and what to do instead. Both example agents had to +#: counter-instruct this in their own prompts, which is a workaround no user +#: should have to discover. This rides `developer_instructions` (additive) +#: rather than `base_instructions`, which would *replace* Codex's ~21KB +#: system prompt. +_TOOL_AVAILABILITY_NOTE = ( + "Tools available on this run:\n" + "- `apply_patch` is not one of them. Create and edit files with " + "`exec_command` instead (for example a `cat > file <<'EOF'` heredoc).\n" + "- Nobody can answer `request_user_input` during this run, and calling it " + "ends your turn with the work undone. Decide with what you have, and say " + "what was missing in your final message." +) class CodexRuntime(BaseRuntime): @@ -134,7 +243,8 @@ async def run_async( shim = await get_shim(api_base, api_key) shim_url = shim.url or "" - workspace, cleanup_workspace = _prepare_workspace(runtime_config, ctx) + workspace = _prepare_workspace(runtime_config, ctx) + await _maybe_reap_workspaces(runtime_config) codex_home = _prepare_codex_home(shim_url, model, runtime_config) # Expose the agent's skills to Codex by materializing them under # `$CODEX_HOME/skills/`, where Codex's native skill system discovers @@ -150,7 +260,9 @@ async def run_async( type(e).__name__, ) - event_queue: asyncio.Queue[object] = asyncio.Queue() + event_queue: asyncio.Queue["Event | BaseException | _QueueSentinel"] = ( + asyncio.Queue() + ) turn_token: str | None = None run_started_at = time.monotonic() run_status = "failed" @@ -159,6 +271,9 @@ async def run_async( async def _emit_tool_event(event: "Event") -> None: await event_queue.put(event) + # Bound to None up front so the `except` below can tell "tools were + # built" from "setup failed before that" without inspecting locals(). + tool_bundle = None try: tool_bundle = await build_executable_tools( agent, @@ -175,23 +290,70 @@ async def _emit_tool_event(event: "Event") -> None: event_sink=_emit_tool_event, timeout_seconds=runtime_config.tool_timeout_seconds, ) - resumed_events = [ - *await resume_authenticated_tools(tool_bundle, ctx), - *await resume_confirmed_tools(tool_bundle, ctx), - ] + # These resume paths execute real tools on *this* task, so the + # workspace is bound around them too: a tool must see the same + # directory whether the shim called it or a resumed confirmation + # did. + with bind_workspace(workspace): + resumed_events = [ + *await resume_authenticated_tools(tool_bundle, ctx), + *await resume_confirmed_tools(tool_bundle, ctx), + ] except BaseException as e: logger.error( "codex_runtime_setup_failed invocation_id=%s stage=tools error_type=%s", ctx.invocation_id, type(e).__name__, ) - if "tool_bundle" in locals(): + if tool_bundle is not None: await close_toolsets(tool_bundle.opened_toolsets) shutil.rmtree(codex_home, ignore_errors=True) - if cleanup_workspace: - shutil.rmtree(workspace, ignore_errors=True) raise + # ADK's `call_llm` span is opened by its own LLM flow, which this + # runtime replaces. Open it here so VeADK's telemetry chain (the + # in-memory exporter's session index, the evaluator, portal metrics and + # the common model-span attributes) sees a Codex invocation at all. + call_llm_span = _start_call_llm_span() + + # One-shot, order-preserving teardown. Three paths reach it (the + # before-model short circuit, the input-setup failure handler and the + # `finally`), and a consumer that stops iterating at the short circuit's + # `yield` raises `GeneratorExit` into the enclosing handler, which then + # runs the `finally` as well. Without the latch that ran `_end_span` + # twice ("Calling end() on an ended span") and closed every MCP toolset + # twice. + cleanup_done = False + + async def _cleanup() -> None: + nonlocal cleanup_done + if cleanup_done: + return + cleanup_done = True + if turn_token is not None: + shim.unregister_turn(turn_token) + await close_toolsets(tool_bundle.opened_toolsets) + # `workspace` is deliberately kept: it is session-scoped and the + # next invocation of this session must see the files this turn + # wrote. See `_prepare_workspace` for its lifetime. + shutil.rmtree(codex_home, ignore_errors=True) + _end_span(call_llm_span) + + # `_emit_call_llm_telemetry`'s contract is one record per invocation: + # the evaluator reads the first span's prompt as the user input and the + # last span's completion as the final answer, and the telemetry layer + # accumulates tokens per span. The normal path emits on completion and + # then yields, so an abandoned consumer used to re-enter the failure + # handler and *overwrite* that record with a `GeneratorExit` error. + telemetry_emitted = False + + def _emit_telemetry_once(llm_response: "LlmResponse") -> None: + nonlocal telemetry_emitted + if telemetry_emitted: + return + telemetry_emitted = True + _emit_call_llm_telemetry(ctx, runtime_call, llm_response, call_llm_span) + try: # Persist resumed confirmation responses before constructing history, # so Codex sees the completed/rejected tool result exactly once. @@ -207,16 +369,7 @@ async def _emit_tool_event(event: "Event") -> None: _scope_event(transferred_event, ctx) yield transferred_event run_status = "transferred" - await close_toolsets(tool_bundle.opened_toolsets) - shutil.rmtree(codex_home, ignore_errors=True) - if cleanup_workspace: - shutil.rmtree(workspace, ignore_errors=True) - logger.info( - "codex_runtime_complete invocation_id=%s status=%s duration_ms=%d", - ctx.invocation_id, - run_status, - round((time.monotonic() - run_started_at) * 1000), - ) + await _cleanup() return runtime_call = await build_runtime_llm_request( @@ -225,11 +378,6 @@ async def _emit_tool_event(event: "Event") -> None: model=model, tools_dict=tool_bundle.tools, ) - append_transfer_instructions( - agent, - runtime_call.llm_request, - transfer_targets, - ) short_circuit = await run_before_model_callbacks( agent, ctx, @@ -243,17 +391,8 @@ async def _emit_tool_event(event: "Event") -> None: runtime_call.model_response_event, ) _scope_event(event, ctx) - await close_toolsets(tool_bundle.opened_toolsets) - shutil.rmtree(codex_home, ignore_errors=True) - if cleanup_workspace: - shutil.rmtree(workspace, ignore_errors=True) - run_status = "completed" - logger.info( - "codex_runtime_complete invocation_id=%s status=%s duration_ms=%d", - ctx.invocation_id, - run_status, - round((time.monotonic() - run_started_at) * 1000), - ) + _emit_telemetry_once(short_circuit) + await _cleanup() yield event return @@ -264,21 +403,68 @@ async def _emit_tool_event(event: "Event") -> None: event_sink=_emit_tool_event, timeout_seconds=runtime_config.tool_timeout_seconds, ) + if "transfer_to_agent" in runtime_call.llm_request.tools_dict: + append_transfer_instructions( + agent, + runtime_call.llm_request, + transfer_targets, + ) turn_token = shim.register_turn( tool_bundle.specs, - tool_bundle.executors, + # Bound here rather than by a ContextVar set for the turn: the + # shim runs executors on a task descended from its uvicorn + # server task, which snapshotted its context when the *first* + # invocation in the process started the shim, so an ambient + # value would be that invocation's - a silent cross-tenant + # leak. See `veadk.runtime.codex.workspace`. + bind_workspace_to_executors(tool_bundle.executors, workspace), max_tool_iterations=runtime_config.max_tool_iterations, invocation_id=ctx.invocation_id, + model_extra_config=agent.model_extra_config, + on_model_call=lambda: _charge_llm_call(ctx), ) # Keep privileged instructions out of the user transcript. The SDK - # exposes native base/developer instruction channels. + # exposes a native developer-instruction channel for them. + # + # `base_instructions` is deliberately NOT used: Codex *replaces* + # its built-in system prompt when it is set (models-manager + # overwrites `instructions_template` and nulls + # `instructions_variables`; nothing concatenates), which would + # delete ~20KB of shipped guidance covering AGENTS.md, planning, + # `update_plan`, `apply_patch` and shell-tool usage. Codex's own + # docs call the equivalent config key strongly discouraged. + # `developer_instructions` is purely additive - Codex renders it as + # its own `developer` message alongside AGENTS.md, skills and + # environment context - so the agent identity block is folded in + # there, ahead of the agent instruction to preserve ordering. prompt = build_prompt_from_llm_request(runtime_call.llm_request) - developer_instructions = system_instruction_to_text( - runtime_call.llm_request.config.system_instruction + developer_instructions = "\n\n".join( + block + for block in ( + (runtime_call.base_instructions or "").strip(), + system_instruction_to_text( + runtime_call.llm_request.config.system_instruction + ).strip(), + _TOOL_AVAILABILITY_NOTE, + ) + if block ) + # Tag the turn's own user message so the shim can tell this turn's + # sampling requests from Codex-internal passes that reuse the same + # provider block and bearer token. Codex re-sends the whole history + # when it auto-compacts, so an untagged shim would advertise the + # agent's ADK tools to the summarizer and replay the turn's tool + # transcript into it - and would then execute a real tool a second + # time if the summarizer asked for one. The tag rides in the prompt + # text rather than a separate input item precisely because Codex + # preserves user-message text verbatim across compaction and + # reordering, where a side-channel item would be dropped. input_items = _build_codex_input( - prompt, runtime_call.llm_request, workspace + prompt, + runtime_call.llm_request, + workspace, + turn_marker=shim.turn_marker(turn_token), ) logger.info( "codex_runtime_start invocation_id=%s agent=%s model=%s " @@ -291,7 +477,26 @@ async def _emit_tool_event(event: "Event") -> None: runtime_config.network_access, len(tool_bundle.executors), ) - + logger.info( + "codex_base_instructions_preserved invocation_id=%s " + "identity_chars=%d developer_chars=%d " + "detail=Codex keeps its built-in system prompt; the agent " + "identity and instruction are sent as developer instructions.", + ctx.invocation_id, + len(runtime_call.base_instructions or ""), + len(developer_instructions), + ) + if runtime_config.approval_mode == "auto_review": + logger.warning( + "codex_approval_auto_accept invocation_id=%s approval_mode=%s " + "detail=Every Codex sandbox escalation and file-change " + "approval request is auto-accepted by the SDK's default " + "approval handler; no human and no ADK confirmation is " + "consulted. Use approval_mode='deny_all' to keep Codex " + "inside the sandbox.", + ctx.invocation_id, + runtime_config.approval_mode, + ) # CodexConfig.env is copied into only this subprocess. Never mutate # process-wide CODEX_HOME or credential variables. sdk_config = CodexConfig( @@ -304,21 +509,25 @@ async def _emit_tool_event(event: "Event") -> None: ctx.invocation_id, type(e).__name__, ) - if turn_token is not None: - shim.unregister_turn(turn_token) - await close_toolsets(tool_bundle.opened_toolsets) - shutil.rmtree(codex_home, ignore_errors=True) - if cleanup_workspace: - shutil.rmtree(workspace, ignore_errors=True) + await _cleanup() raise turn = None pump: asyncio.Task[None] | None = None + # Lookahead for the tool-only turn. That turn's merged response carries + # the turn's `usage_metadata` and any `state_delta` a model callback + # wrote, but it has no content, and a contentless, tool-free, + # non-partial event reads as the invocation's final response + # (`Event.is_final_response()`) - so it cannot simply be emitted. The + # last *durable* event is therefore held back to give that bookkeeping + # somewhere real to land; see `_merge_turn_bookkeeping`. Partial events + # are no use as a target (they are never persisted), so any that follow + # the held-back one are buffered behind it rather than overtaking it. + merge_target: "Event | None" = None try: async with AsyncCodex(config=sdk_config) as codex: thread = await codex.thread_start( model=model, model_provider=_PROVIDER_ID, - base_instructions=runtime_call.base_instructions or None, developer_instructions=developer_instructions or None, cwd=workspace, ephemeral=True, @@ -334,17 +543,23 @@ async def _emit_tool_event(event: "Event") -> None: effort=ReasoningEffort(runtime_config.reasoning_effort), ) stream = turn.stream() + # Latest `ThreadTokenUsageUpdatedNotification` payload. The + # thread is created fresh and ephemeral for this invocation, so + # its `total` breakdown is this invocation's complete usage. + latest_token_usage: dict[str, Any] = {} async def _pump_codex() -> None: active_tool_items: set[str] = set() try: + # No turn-id filtering here: `AsyncTurnHandle.stream()` + # reads a per-turn queue that `MessageRouter` already + # fills strictly by turn id, so every notification on + # this stream belongs to `turn`. The previous filter was + # both redundant and wrong - `TurnStartedNotification` + # carries no `turn_id`, so it was only ever skipped by + # accident of the attribute being absent. async for note in stream: payload = note.payload - payload_turn_id = getattr(payload, "turn_id", None) - if isinstance(payload, TurnCompletedNotification): - payload_turn_id = payload.turn.id - if payload_turn_id and payload_turn_id != turn.id: - continue for event in notification_to_events( payload, agent.name, @@ -357,10 +572,14 @@ async def _pump_codex() -> None: and event.custom_metadata.get("codex_event_type") == "token_usage" ): + usage = event.custom_metadata.get("token_usage") + if isinstance(usage, dict): + latest_token_usage.clear() + latest_token_usage.update(usage) logger.info( "codex_token_usage invocation_id=%s usage=%s", ctx.invocation_id, - event.custom_metadata.get("token_usage"), + usage, ) await event_queue.put(event) except BaseException as e: @@ -372,66 +591,135 @@ async def _pump_codex() -> None: await event_queue.put(_QUEUE_DONE) pump = asyncio.create_task(_pump_codex()) - buffer_final_text = has_after_model_callbacks(agent, ctx) + # Buffer unconditionally. Codex emits one durable `agentMessage` + # per intermediate model reply, so streaming them straight + # through would produce several `is_final_response()` events and + # make `output_key`, evaluation and the A2A reply + # last-writer-wins on whichever preamble arrived last. Buffering + # only when an after-model callback happens to be registered + # also made the event shape depend on plugin installation. final_text_events: list[Event] = [] - transfer_requested = False - deferred_transfer_event: Event | None = None while True: queued = await event_queue.get() if queued is _QUEUE_DONE: break if isinstance(queued, BaseException): raise queued - event = queued # type: ignore[assignment] - transfer_target = transfer_agent_name(event) - if transfer_target and use_adk_transfer_scheduler: - transfer_requested = True - run_status = "transferred" - final_text_events.clear() - deferred_transfer_event = event - continue - if buffer_final_text and is_final_model_text_event( - event, agent.name - ): + event = queued + if is_codex_final_text_event(event): final_text_events.append(event) continue - yield event + # Partials go out immediately, even while a durable event + # is held back as the merge target. Parking them behind it + # stalled the live stream for the rest of the turn: the + # final answer's deltas and a command's output both arrive + # after the last durable event, so they were delivered only + # once the Codex stream had already ended. Overtaking is + # safe because partials are never persisted + # (`BaseSessionService.append_event` returns early on them), + # so only the order among durable events is observable in + # session history, and that order is unchanged. + if event.partial: + yield event + continue + transfer_target = transfer_agent_name(event) if transfer_target: - transfer_requested = True + if merge_target is not None: + yield merge_target + merge_target = None final_text_events.clear() + yield event + run_status = "transferred" + if use_adk_transfer_scheduler: + return async for transferred_event in run_transferred_agent( ctx, event ): _scope_event(transferred_event, ctx) yield transferred_event - run_status = "transferred" - break - if transfer_requested: - await pump - if deferred_transfer_event is not None: - yield deferred_transfer_event - return + return + if merge_target is not None: + yield merge_target + merge_target = event await pump - if final_text_events: - llm_response = final_events_to_llm_response(final_text_events) - llm_response = await run_after_model_callbacks( - agent, - ctx, - llm_response, - runtime_call.model_response_event, - ) - event = llm_response_to_event( - runtime_call.llm_request, - llm_response, - runtime_call.model_response_event, - ) - _scope_event(event, ctx) + + # The shim serves backend calls on the server's task, so an + # exception it raised (an exhausted `max_llm_calls` budget) got + # relayed to Codex as a 429 and recorded rather than propagated. + # Re-raise it here so Runner's normal handling still applies, + # instead of returning whatever partial answer Codex salvaged. + shim_error = shim.turn_error(turn_token) + if shim_error is not None: + raise shim_error + + # One merged response per turn, always: after-model callbacks + # must run on every turn (ADK does, and the harness collects + # token usage only through them), so this is not gated on there + # being text to emit. + llm_response = final_events_to_llm_response(final_text_events) + usage_metadata = build_turn_usage_metadata(latest_token_usage) + if usage_metadata is not None: + llm_response.usage_metadata = usage_metadata + llm_response = await run_after_model_callbacks( + agent, + ctx, + llm_response, + runtime_call.model_response_event, + ) + _emit_telemetry_once(llm_response) + event = llm_response_to_event( + runtime_call.llm_request, + llm_response, + runtime_call.model_response_event, + ) + _scope_event(event, ctx) + if event.content and event.content.parts: + if merge_target is not None: + yield merge_target + merge_target = None + yield event + elif merge_target is not None: + # A tool-only turn: the merged event has no text, and a + # contentless, tool-free, non-partial event satisfies + # `Event.is_final_response()` - a spurious "the agent + # answered" marker for any consumer keying on that alone, + # including upstream ADK code that stops at the first final + # response. (VeADK's own readers of the final answer - + # `runtime/output_state.py`, `evaluation/base_evaluator.py`, + # `runner.py`'s A2A path - all guard on content as well, so + # they are not what is at risk here.) Dropping it whole, + # however, also threw away the + # `state_delta` model callbacks wrote through + # `CallbackContext(ctx, event_actions=model_response_event.actions)` + # and the turn's `usage_metadata`. Marking it partial does + # not rescue either: partial events are never persisted + # (`google/adk/sessions/base_session_service.py`). So the + # bookkeeping is folded onto the last tool event instead - + # an event that is emitted, persisted, and is not a final + # response. + merge_turn_bookkeeping(merge_target, event) + yield merge_target + merge_target = None + else: + # Nothing durable was emitted this turn: there is nowhere + # else for the bookkeeping to go, and no earlier event that + # this one's final-response marker could displace. yield event run_status = "completed" except asyncio.CancelledError: - if run_status != "transferred": - run_status = "cancelled" - if turn is not None and run_status != "transferred": + run_status = "cancelled" + # A recorded shim error is deliberately *not* substituted here. + # `CancelledError` must reach the awaiting task unchanged or the + # cancellation is swallowed and asyncio's contract is broken; the + # budget error is logged instead so the cause is still visible. + if shim.turn_error(turn_token) is not None: + logger.warning( + "codex_shim_turn_error_dropped_on_cancel invocation_id=%s " + "error_type=%s", + ctx.invocation_id, + type(shim.turn_error(turn_token)).__name__, + ) + if turn is not None: try: await turn.interrupt() except Exception: # noqa: BLE001 @@ -441,12 +729,44 @@ async def _pump_codex() -> None: ) raise except BaseException as e: + # Read the shim's recorded error *before* the `finally`'s + # `unregister_turn` drops the turn state. The shim serves backend + # calls on the server's task, so an exhausted `max_llm_calls` budget + # cannot propagate from there: it is recorded, relayed to Codex as a + # failed response, and re-read once the turn ends. That read used to + # live on the success path only, so any transport failure arriving + # afterwards - the pump re-raising, the Codex SDK erroring - jumped + # straight here and the budget error was silently discarded, taking + # the whole `max_llm_calls` feature with it and handing the + # `on_model_error` callbacks the wrong exception. The shim error is + # the *cause* and wins; the transport failure is chained onto it. + shim_error = shim.turn_error(turn_token) + if shim_error is not None and shim_error is not e: + logger.warning( + "codex_shim_turn_error_preferred invocation_id=%s " + "shim_error_type=%s transport_error_type=%s", + ctx.invocation_id, + type(shim_error).__name__, + type(e).__name__, + ) + if shim_error.__cause__ is None: + shim_error.__cause__ = e + e = shim_error logger.error( "codex_runtime_failed invocation_id=%s error_type=%s", ctx.invocation_id, type(e).__name__, ) - if isinstance(e, Exception) and "runtime_call" in locals(): + # Nothing already streamed may be lost to the failure. Never on a + # `GeneratorExit`: the consumer has stopped reading, and yielding + # while it propagates is a hard `RuntimeError`. + if not isinstance(e, GeneratorExit): + held, merge_target = merge_target, None + if held is not None: + yield held + # `runtime_call` is always bound here: it is assigned in the + # preceding block, whose handler re-raises on failure. + if isinstance(e, Exception): fallback = await run_on_model_error_callbacks( agent, ctx, @@ -461,20 +781,21 @@ async def _pump_codex() -> None: runtime_call.model_response_event, ) _scope_event(event, ctx) + _emit_telemetry_once(fallback) yield event run_status = "completed" return - raise + # A `GeneratorExit` means the consumer stopped iterating, not that + # the model failed: recording it would overwrite a completed span's + # attributes with an error that never happened. + if not isinstance(e, GeneratorExit): + _emit_telemetry_once(_error_llm_response(e)) + raise e finally: if pump is not None and not pump.done(): pump.cancel() await asyncio.gather(pump, return_exceptions=True) - if turn_token is not None: - shim.unregister_turn(turn_token) - await close_toolsets(tool_bundle.opened_toolsets) - shutil.rmtree(codex_home, ignore_errors=True) - if cleanup_workspace: - shutil.rmtree(workspace, ignore_errors=True) + await _cleanup() logger.info( "codex_runtime_complete invocation_id=%s status=%s duration_ms=%d", ctx.invocation_id, @@ -519,7 +840,10 @@ def _prepare_codex_home( f"review_model = {toml_string(model)}\n" f"approval_policy = {toml_string(approval_policy)}\n" f"sandbox_mode = {toml_string(sandbox_mode)}\n" - f"disable_response_storage = true\n" + # `disable_response_storage` is intentionally absent: it was removed + # upstream (absent from config.schema.json and from the pinned CLI + # binary) and `store: false` is now unconditional in Codex's client, + # so writing it here only produced a silently ignored key. f"model_reasoning_effort = {toml_string(runtime_config.reasoning_effort)}\n" f"personality = {toml_string(runtime_config.personality)}\n\n" f"[model_providers.{_PROVIDER_ID}]\n" @@ -538,11 +862,37 @@ def _prepare_codex_home( def _prepare_workspace( runtime_config: CodexRuntimeConfig, ctx: "InvocationContext" -) -> tuple[str, bool]: +) -> str: + """Resolve the filesystem Codex is given as its ``cwd``. + + The workspace is keyed by app/user/session/agent, so successive + invocations of one session share it and it is never deleted at the end of + a turn. Process-owned workspaces are removed by the ``atexit`` hook + :func:`_ensure_session_workspace_root` registers and, while the process + runs, by :func:`_reap_idle_workspaces`. + + An ADK tool that needs to write into this directory reads it back with + :func:`veadk.runtime.codex.current_workspace`; see + :mod:`veadk.runtime.codex.workspace` for why that is bound per tool call. + + A caller-supplied ``workspace_root`` is never reaped, because this runtime + cannot tell its own session directories from whatever else the caller keeps + there. **Cleaning it is therefore the caller's responsibility**: a + long-lived server that sets ``workspace_root`` accumulates one directory + per session indefinitely. Leave ``workspace_root`` unset to get the + reaped, process-owned root instead. + + Args: + runtime_config (CodexRuntimeConfig): Resolved runtime configuration. + ctx (InvocationContext): The invocation being served. + + Returns: + str: Absolute path to the workspace directory. + """ root = runtime_config.workspace_root if root and runtime_config.reuse_workspace: Path(root).mkdir(parents=True, exist_ok=True) - return root, False + return root session = getattr(ctx, "session", None) session_id = str(getattr(session, "id", "session")) @@ -556,12 +906,212 @@ def _prepare_workspace( ) digest = hashlib.sha256(scope.encode("utf-8")).hexdigest()[:16] safe_id = "".join(ch for ch in session_id if ch.isalnum() or ch in "-_")[:32] - base = Path(root or _SESSION_WORKSPACE_ROOT) + base = Path(root or _ensure_session_workspace_root()) base.mkdir(parents=True, exist_ok=True) workspace = base / f"{safe_id or 'session'}-{digest}" workspace.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(workspace, 0o700) - return str(workspace), False + # Mark the session active so the reaper keeps it for another TTL window. + os.utime(workspace) + return str(workspace) + + +async def _maybe_reap_workspaces(runtime_config: CodexRuntimeConfig) -> None: + """Run the idle-workspace reaper off the event loop, at most periodically. + + The reaper is all blocking syscalls — ``iterdir``, ``stat`` and recursive + ``rmtree`` — and it used to run inline in :func:`_prepare_workspace`, before + the invocation's first ``await``. In a server that stalled *every* other + in-flight turn once per invocation. It now runs in a worker thread, no more + than once per :data:`_WORKSPACE_REAP_INTERVAL_SECONDS`, and removes at most + :data:`_WORKSPACE_REAP_MAX_PER_PASS` trees per pass. + + Only the process-owned root is ever reaped: a caller-supplied + ``workspace_root`` may hold data this runtime does not own. + + Args: + runtime_config (CodexRuntimeConfig): Resolved runtime configuration. + """ + global _last_workspace_reap_at + if runtime_config.workspace_root: + return + # Read, never create: with no root yet there is nothing to reap, and + # materializing one here would reintroduce exactly the stray directory the + # lazy root exists to avoid. + root = _session_workspace_root + if root is None: + return + now = time.monotonic() + with _workspace_reap_lock: + if now - _last_workspace_reap_at < _WORKSPACE_REAP_INTERVAL_SECONDS: + return + _last_workspace_reap_at = now + try: + await asyncio.to_thread(_reap_idle_workspaces, Path(root)) + except Exception: # noqa: BLE001 - housekeeping must never fail a turn + logger.warning("codex_workspace_reap_failed") + + +def _reap_idle_workspaces(base: Path) -> None: + """Delete session workspaces untouched for the idle TTL. + + Best-effort: a workspace that cannot be inspected or removed is left in + place rather than failing the invocation. Runs on a worker thread; see + :func:`_maybe_reap_workspaces`. + + Args: + base (Path): The process-owned session workspace root to scan. + """ + cutoff = time.time() - _WORKSPACE_IDLE_TTL_SECONDS + try: + entries = list(base.iterdir()) + except OSError: + return + reaped = 0 + for entry in entries: + if reaped >= _WORKSPACE_REAP_MAX_PER_PASS: + logger.info( + "codex_workspace_reap_truncated limit=%d", _WORKSPACE_REAP_MAX_PER_PASS + ) + return + try: + if not entry.is_dir() or entry.stat().st_mtime >= cutoff: + continue + except OSError: + continue + shutil.rmtree(entry, ignore_errors=True) + reaped += 1 + logger.info("codex_workspace_reaped workspace=%s", entry.name) + + +def _start_call_llm_span() -> "Span | None": + """Open the ADK-shaped ``call_llm`` span for one Codex invocation. + + VeADK keys its whole model-telemetry chain off a span literally named + ``call_llm`` in ADK's tracer scope: the in-memory exporter indexes sessions + by it, the evaluator reads its prompt/completion attributes, and portal + metrics and the common model-span attributes are written from + :func:`veadk.tracing.telemetry.telemetry.trace_call_llm`. ADK opens that + span inside the LLM flow this runtime replaces, so the runtime must open it + itself. + + ``start_span`` is used rather than ``start_as_current_span``: ``run_async`` + is an async generator, so a context manager spanning its ``yield`` points + would attach the OTel context in one task resumption and detach it in + another, corrupting the context stack. Keeping the span non-current also + leaves tool spans as siblings of ``call_llm`` under ``invoke_agent``, which + is ADK's own shape. + + Returns: + Span | None: The started span, or ``None`` when tracing is unavailable. + """ + try: + from google.adk.telemetry.tracing import tracer + + return tracer.start_span("call_llm") + except Exception: # noqa: BLE001 + logger.warning("codex_trace_span_start_failed") + return None + + +def _end_span(span: "Span | None") -> None: + """End a span without ever failing the turn.""" + if span is None: + return + try: + span.end() + except Exception: # noqa: BLE001 + logger.warning("codex_trace_span_end_failed") + + +def _emit_call_llm_telemetry( + ctx: "InvocationContext", + runtime_call: RuntimeLlmCall, + llm_response: "LlmResponse", + span: "Span | None", +) -> None: + """Write one turn's model telemetry onto the ``call_llm`` span. + + Emitted exactly once per invocation - not once per backend HTTP call - + because the evaluator reads the first span's prompt as the user input and + the last span's completion as the final answer, and the telemetry layer + accumulates tokens per span. Several spans would therefore surface Codex's + internally built prompt as the user input and double-count usage. + + The span is made current only for this synchronous call, since portal + metrics derive the call duration from the current span's start time. + + Args: + ctx (InvocationContext): The invocation being served. + runtime_call (RuntimeLlmCall): The request built for this invocation. + llm_response (LlmResponse): The merged response for this turn. + span (Span | None): The owning ``call_llm`` span, if tracing is active. + """ + if span is None: + return + try: + from opentelemetry import trace as otel_trace + + from veadk.tracing.telemetry.telemetry import trace_call_llm + + with otel_trace.use_span(span, end_on_exit=False): + # `trace_call_llm` annotates `span` as `opentelemetry.sdk.trace.Span`, + # but it only ever uses the API-level surface on it (`set_attribute`, + # `.context`) and assigns `trace.get_current_span()` - an API span - + # into the same parameter on its own `span is None` path. The API + # type is what actually arrives here: with no SDK TracerProvider + # configured, ADK's tracer is a `ProxyTracer` returning a + # `NonRecordingSpan`, which is not an SDK `Span`. Keep the accurate + # annotation and ignore the over-narrow one upstream. + trace_call_llm( + ctx, + runtime_call.model_response_event.id, + runtime_call.llm_request, + llm_response, + span, # type: ignore[arg-type] + ) + except Exception: # noqa: BLE001 + logger.warning( + "codex_trace_call_llm_failed invocation_id=%s", + getattr(ctx, "invocation_id", ""), + ) + + +def _error_llm_response(error: BaseException) -> "LlmResponse": + """Build the response recorded on the span when a turn fails.""" + from google.adk.models.llm_response import LlmResponse + + return LlmResponse( + error_code=type(error).__name__, + error_message=str(error) or type(error).__name__, + ) + + +def _charge_llm_call(ctx: "InvocationContext") -> None: + """Charge one backend model call to the invocation's ADK call budget. + + ADK enforces ``RunConfig.max_llm_calls`` solely from + ``InvocationContext.increment_llm_call_count``, which only its own + ``BaseLlmFlow`` calls. The Codex runtime replaces that flow, so without + this hook ``max_llm_calls`` - and ``Runner``'s ``LlmCallsLimitExceededError`` + handling - never fire for ``runtime="codex"``. + + It is handed to the shim as ``register_turn(on_model_call=...)`` so every + backend call Codex's inner loop makes is charged, not just one per turn. + The shim serves those calls on its own task, so a raise cannot propagate + here: it is recorded on the turn state, returned to Codex as a ``429``, and + re-raised by ``run_async`` once the turn ends. + + Args: + ctx (InvocationContext): The invocation being served. + + Raises: + google.adk.agents.invocation_context.LlmCallsLimitExceededError: When + the invocation exceeds ``RunConfig.max_llm_calls``. + """ + increment = getattr(ctx, "increment_llm_call_count", None) + if callable(increment): + increment() def _approval_mode(config: CodexRuntimeConfig) -> ApprovalMode: @@ -581,8 +1131,22 @@ def _sandbox(config: CodexRuntimeConfig) -> Sandbox: def _build_codex_input( - prompt: str, llm_request: "LlmRequest", workspace: str + prompt: str, + llm_request: "LlmRequest", + workspace: str, + *, + turn_marker: str = "", ) -> list[object]: + """Build the Codex turn input, tagging it with the shim's turn marker. + + The marker is appended to the prompt text (a single machine-shaped line, in + the same register as Codex's own ```` blocks) rather + than sent as an extra item: the Codex prompt is the one channel guaranteed + to reach the model request intact, and its text survives Codex's own + compaction, which rebuilds history from user-message text alone. + """ + if turn_marker: + prompt = f"{prompt}\n\n{turn_marker}" items: list[object] = [TextInput(prompt)] for attachment in build_input_attachments_from_llm_request(llm_request, workspace): kind = attachment["kind"] diff --git a/veadk/runtime/codex/translate.py b/veadk/runtime/codex/translate.py index 626c8bebc..757e8cd86 100644 --- a/veadk/runtime/codex/translate.py +++ b/veadk/runtime/codex/translate.py @@ -25,6 +25,7 @@ import json import mimetypes import os +from collections.abc import Callable from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Any @@ -207,6 +208,85 @@ def build_content_attachments(content: Any, workspace: str) -> list[dict[str, st return attachments +def _token_count(breakdown: dict[str, Any], *names: str) -> int | None: + """Read one integer token counter, tolerating snake_case or camelCase.""" + for name in names: + value = breakdown.get(name) + if isinstance(value, int) and not isinstance(value, bool): + return value + return None + + +def build_usage_metadata( + breakdown: Any, +) -> types.GenerateContentResponseUsageMetadata | None: + """Map one Codex ``TokenUsageBreakdown`` onto genai usage metadata. + + Codex counters nest: ``cached_input_tokens`` is part of ``input_tokens`` + and ``reasoning_output_tokens`` is part of ``output_tokens``. genai's + ``thoughts_token_count`` is instead *disjoint* from + ``candidates_token_count`` -- its total is + ``prompt + candidates + tool_use_prompt + thoughts`` -- so reasoning tokens + are deliberately left unmapped; carrying them would double-count for any + consumer that recomputes a total. The reasoning figure stays readable on + ``custom_metadata["token_usage"]``. This mirrors the mapping already used + for Ark responses in :mod:`veadk.models.ark_llm`. + + Args: + breakdown (Any): A ``TokenUsageBreakdown``-shaped mapping. + + Returns: + google.genai.types.GenerateContentResponseUsageMetadata | None: The + mapped usage, or ``None`` when the breakdown is missing or carries no + usable counter, so a malformed payload degrades to "no accounting" + rather than to zeroed accounting that would pollute token histograms. + """ + if not isinstance(breakdown, dict): + return None + prompt = _token_count(breakdown, "input_tokens", "inputTokens") + candidates = _token_count(breakdown, "output_tokens", "outputTokens") + total = _token_count(breakdown, "total_tokens", "totalTokens") + cached = _token_count(breakdown, "cached_input_tokens", "cachedInputTokens") + if prompt is None and candidates is None and total is None: + return None + if total is None: + total = (prompt or 0) + (candidates or 0) + return types.GenerateContentResponseUsageMetadata( + prompt_token_count=prompt, + candidates_token_count=candidates, + total_token_count=total, + cached_content_token_count=cached, + ) + + +def build_turn_usage_metadata( + token_usage: Any, +) -> types.GenerateContentResponseUsageMetadata | None: + """Map a ``ThreadTokenUsage`` mapping's cumulative ``total`` breakdown. + + ``thread/tokenUsage/updated`` carries both ``last`` (the model call that + just finished) and ``total`` (cumulative for the thread). Because the Codex + thread is created fresh and ephemeral for each ADK invocation, the final + ``total`` *is* that invocation's complete usage. Callers should therefore + attach this to the single merged final response instead of summing ``last`` + across the per-notification lifecycle events: those events are ``partial`` + and so are never persisted, which would leave a live stream and a reloaded + session reporting different totals. ``last`` is used only as a fallback + when ``total`` is absent; for a single-round turn the two are identical. + + Args: + token_usage (Any): The mapping published on a ``token_usage`` + lifecycle event's ``custom_metadata["token_usage"]``. + + Returns: + google.genai.types.GenerateContentResponseUsageMetadata | None: The + cumulative usage, or ``None`` when it cannot be read. + """ + if not isinstance(token_usage, dict): + return None + return build_usage_metadata(token_usage.get("total") or token_usage.get("last")) + + def _same_content(left: Any, right: Any) -> bool: if left is right: return True @@ -403,7 +483,9 @@ def item_to_events(item: Any, author: str, invocation_id: str) -> list[Event]: - tool calls (``commandExecution`` / ``mcpToolCall`` / ``dynamicToolCall`` / ``fileChange`` / ``webSearch``) -> a ``function_call`` part plus a matching ``function_response`` part carrying the tool's output, - - ``agentMessage`` / ``plan`` / any other text-bearing item -> a text part, + - ``agentMessage`` -> a durable model text part (the assistant's answer), + - any other text-bearing item (notably ``plan``) -> a ``partial`` + ``plan_item`` lifecycle event carrying the text, - ``userMessage`` (and anything else) -> nothing. Returning per-item keeps the conversion reusable both for the streaming @@ -453,94 +535,175 @@ def item_to_events(item: Any, author: str, invocation_id: str) -> list[Event]: ), ] - if itype != "userMessage" and data.get("text"): + text = data.get("text") + if itype == "agentMessage" and text: + return [_event(author, invocation_id, "model", types.Part(text=str(text)))] + + if itype != "userMessage" and text: + # Narration, not an answer. ``plan`` items carry ``text`` too, and + # emitting those durably lets plan chatter clobber ``output_key`` and + # the A2A reply. Keep them visible but partial, so they stream to the + # UI and stay out of session history. return [ - _event(author, invocation_id, "model", types.Part(text=str(data["text"]))) + _lifecycle_event( + author, + invocation_id, + "plan_item", + {"item_id": data.get("id"), "item_type": itype}, + part=types.Part(text=str(text)), + partial=True, + ) ] return [] -def notification_to_events( - payload: Any, - author: str, - invocation_id: str, - *, - active_tool_items: set[str] | None = None, -) -> list[Event]: - """Translate a Codex lifecycle notification into observable ADK events. +def is_codex_final_text_event(event: Event) -> bool: + """Report whether an event is Codex's durable assistant answer. - Completed items still use :func:`item_to_events`, while starts, output - deltas, plan changes, approval reviews, turn completion, and errors carry a - stable ``custom_metadata.codex_event_type`` for Trace/UI consumers. + True only for a completed ``agentMessage`` item that carries visible + (non-thought) text. Lifecycle markers, reasoning, plan narration and tool + traffic are all excluded, so callers can buffer or post-process the real + answer without re-deriving intent from :meth:`Event.is_final_response`. + + Args: + event (google.adk.events.event.Event): A translated Codex event. + + Returns: + bool: Whether this event holds the turn's assistant answer. """ - data = _item_dict(payload) - kind = type(payload).__name__ - active_tool_items = active_tool_items if active_tool_items is not None else set() - - if kind == "ItemStartedNotification": - item = data.get("item") or {} - item_id = str(item.get("id") or "") - call = _tool_call(item) - if call is not None: - name, args, _ = call - active_tool_items.add(item_id) - return [ - _lifecycle_event( - author, - invocation_id, - "item_started", - { - "item_id": item_id, - "item_type": item.get("type"), - "status": "in_progress", - }, - part=types.Part( - function_call=types.FunctionCall( - id=item_id, name=name, args=args - ) - ), - ) - ] - return [ - _lifecycle_event( - author, - invocation_id, - "item_started", - { - "item_id": item_id, - "item_type": item.get("type"), - "status": "in_progress", - }, - ) - ] + metadata = event.custom_metadata or {} + if metadata.get("codex_event_type") != "item_completed": + return False + if metadata.get("item_type") != "agentMessage": + return False + content = event.content + if content is None: + return False + return any( + part.text is not None and not getattr(part, "thought", False) + for part in content.parts or [] + ) + + +_ERROR_CODE_FALLBACK = "codex_error" + + +def _camel(value: str) -> str: + """Normalize a ``snake_case`` identifier to the SDK's camelCase codes.""" + head, *rest = value.split("_") + return head + "".join(word[:1].upper() + word[1:] for word in rest) + + +def _error_code(error: Any) -> str: + """Classify a Codex ``TurnError`` into a stable ``Event.error_code``. + + ``TurnError`` is ``{message, additional_details, codex_error_info}`` -- it + has no ``code`` field, so the machine-readable classification has to come + from ``codex_error_info``. That value is either a bare enum + (``"contextWindowExceeded"``) or a single-key object naming the variant + (``{"http_connection_failed": {"http_status_code": 503}}``); both normalize + to the camelCase code the SDK documents. A literal ``code`` key is still + honoured afterwards for hand-built dict payloads. + + Args: + error (Any): The error mapping carried by the notification. - if kind == "ItemCompletedNotification": - item = data.get("item") or {} - item_id = str(item.get("id") or "") - converted = item_to_events(item, author, invocation_id) - if item_id in active_tool_items and len(converted) == 2: - converted = converted[1:] - active_tool_items.discard(item_id) - for event in converted: - event.custom_metadata = { - "codex_event_type": "item_completed", - "item_id": item_id, - "item_type": item.get("type"), - "status": _scalar(item.get("status")) or "completed", - } - return converted - - delta_types = { - "AgentMessageDeltaNotification": "message_delta", - "CommandExecutionOutputDeltaNotification": "command_output", - "FileChangeOutputDeltaNotification": "file_change_output", - "McpToolCallProgressNotification": "mcp_progress", - "PlanDeltaNotification": "plan_delta", - "ReasoningSummaryTextDeltaNotification": "reasoning_delta", - "ReasoningTextDeltaNotification": "reasoning_delta", + Returns: + str: A stable error code, or ``"codex_error"`` when unclassifiable. + """ + if not isinstance(error, dict): + return _ERROR_CODE_FALLBACK + info = _scalar(error.get("codex_error_info") or error.get("codexErrorInfo")) + if isinstance(info, str) and info: + return _camel(info) + if isinstance(info, dict): + for key in info: + if key: + return _camel(str(key)) + code = error.get("code") + if code: + return str(_scalar(code)) + return _ERROR_CODE_FALLBACK + + +# Every handler receives the dumped payload, the invocation scope, and the +# mutable set of tool item ids whose ``function_call`` part was already emitted +# at item start; it returns the ADK events for that one notification. +_NotificationHandler = Callable[[dict[str, Any], str, str, set[str]], list[Event]] + + +def _on_item_started( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Announce a thread item; tool items also carry their ``function_call``.""" + item = data.get("item") or {} + item_id = str(item.get("id") or "") + metadata = { + "item_id": item_id, + "item_type": item.get("type"), + "status": "in_progress", } - if kind in delta_types: + call = _tool_call(item) + if call is None: + return [_lifecycle_event(author, invocation_id, "item_started", metadata)] + name, args, _ = call + # Remember the id so the completed item emits only the response half. An + # id-less item is never tracked: every one of them would share the empty + # key, so the *next* id-less tool item would match and lose its + # `function_call`, leaving an orphan `function_response` in the session. + if item_id: + active_tool_items.add(item_id) + return [ + _lifecycle_event( + author, + invocation_id, + "item_started", + metadata, + part=types.Part( + function_call=types.FunctionCall(id=item_id, name=name, args=args) + ), + # A tool call/response pair has to persist, and the call part + # already keeps the event out of ``is_final_response``. + partial=None, + ) + ] + + +def _on_item_completed( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Emit the finished item, dropping a ``function_call`` already announced.""" + item = data.get("item") or {} + item_id = str(item.get("id") or "") + converted = item_to_events(item, author, invocation_id) + if item_id and item_id in active_tool_items and len(converted) == 2: + converted = converted[1:] + active_tool_items.discard(item_id) + for event in converted: + # Preserve a narrower type already assigned by ``item_to_events`` + # (``plan_item``); everything else is a plain completed item. + existing = event.custom_metadata or {} + event.custom_metadata = { + "codex_event_type": existing.get("codex_event_type") or "item_completed", + "item_id": item_id, + "item_type": item.get("type"), + "status": _scalar(item.get("status")) or "completed", + } + return converted + + +def _delta_handler(event_type: str, *, thought: bool = False) -> _NotificationHandler: + """Build the handler for one streaming-delta notification family.""" + + def handler( + data: dict[str, Any], + author: str, + invocation_id: str, + active_tool_items: set[str], + ) -> list[Event]: + # ``McpToolCallProgressNotification`` names its text field ``message``; + # every other member of the family names it ``delta``. text = str(data.get("delta") or data.get("message") or "") if not text: return [] @@ -548,80 +711,117 @@ def notification_to_events( _lifecycle_event( author, invocation_id, - delta_types[kind], + event_type, {"item_id": data.get("item_id"), "status": "in_progress"}, - part=types.Part( - text=text, - thought=kind - in { - "ReasoningSummaryTextDeltaNotification", - "ReasoningTextDeltaNotification", - }, - ), + part=types.Part(text=text, thought=thought), partial=True, ) ] - if kind == "FileChangePatchUpdatedNotification": - return [ - _lifecycle_event( - author, - invocation_id, - "file_change_patch", - { - "item_id": data.get("item_id"), - "changes": data.get("changes") or [], - "status": "in_progress", - }, - partial=True, - ) - ] + return handler - if kind == "TurnPlanUpdatedNotification": - return [ - _lifecycle_event( - author, - invocation_id, - "plan_update", - { - "explanation": data.get("explanation"), - "plan": data.get("plan") or [], - }, - ) - ] - if kind == "TurnStartedNotification": - turn = data.get("turn") or {} - return [ - _lifecycle_event( - author, - invocation_id, - "turn_started", - { - "turn_id": turn.get("id"), - "status": _scalar(turn.get("status")) or "in_progress", - }, - ) - ] +def _on_file_change_patch( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Stream the in-progress patch for a ``fileChange`` item.""" + return [ + _lifecycle_event( + author, + invocation_id, + "file_change_patch", + { + "item_id": data.get("item_id"), + "changes": data.get("changes") or [], + "status": "in_progress", + }, + partial=True, + ) + ] - if kind == "ThreadTokenUsageUpdatedNotification": - return [ - _lifecycle_event( - author, - invocation_id, - "token_usage", - { - "turn_id": data.get("turn_id"), - "token_usage": data.get("token_usage") or {}, - }, - ) - ] - if kind in { - "ItemGuardianApprovalReviewStartedNotification", - "ItemGuardianApprovalReviewCompletedNotification", - }: - status = "in_progress" if kind.endswith("StartedNotification") else "completed" +def _on_turn_plan_updated( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Surface the agent's updated to-do plan for the turn.""" + return [ + _lifecycle_event( + author, + invocation_id, + "plan_update", + { + "explanation": data.get("explanation"), + "plan": data.get("plan") or [], + }, + ) + ] + + +def _on_turn_started( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Mark the start of a turn. + + The payload is ``{thread_id, turn}``; there is no top-level ``turn_id``, so + the id is read from the nested ``turn``. + """ + turn = data.get("turn") or {} + return [ + _lifecycle_event( + author, + invocation_id, + "turn_started", + { + "turn_id": turn.get("id"), + "status": _scalar(turn.get("status")) or "in_progress", + }, + ) + ] + + +def _on_token_usage( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Publish Codex token accounting as an observable lifecycle event. + + The raw SDK mapping -- ``last``, ``total`` and ``model_context_window`` -- + is kept verbatim on ``custom_metadata["token_usage"]``, which is the shape + the UI already reads and the only place ``reasoning_output_tokens`` + survives. + + Deliberately no ``usage_metadata`` here. This notification fires once per + model call, and every consumer of ``usage_metadata`` sums it across events + with no dedupe, so putting ``last`` on each of these would be correct only + while they are all delivered -- and they are ``partial``, hence never + persisted, so a reloaded session would disagree with the live stream. + ``after_model_callback`` collectors such as the harness usage plugin would + never see them at all. The runtime instead attaches the cumulative figure + once, via :func:`build_turn_usage_metadata`, to the merged final response; + the Codex thread is ephemeral per invocation, so that ``total`` is exactly + this invocation's usage. + """ + return [ + _lifecycle_event( + author, + invocation_id, + "token_usage", + { + "turn_id": data.get("turn_id"), + "token_usage": data.get("token_usage") or {}, + }, + ) + ] + + +def _approval_handler(status: str) -> _NotificationHandler: + """Build the handler for one side of a guardian approval review.""" + + def handler( + data: dict[str, Any], + author: str, + invocation_id: str, + active_tool_items: set[str], + ) -> list[Event]: return [ _lifecycle_event( author, @@ -637,41 +837,199 @@ def notification_to_events( ) ] - if kind == "ErrorNotification": - error = data.get("error") or {} - message = str(error.get("message") or error) - return [ - Event( - invocation_id=invocation_id, - author=author, - error_code=str(error.get("code") or "codex_error"), - error_message=message, - custom_metadata={ - "codex_event_type": "error", - "will_retry": bool(data.get("will_retry")), - }, - ) - ] + return handler - if kind == "TurnCompletedNotification": - turn = data.get("turn") or {} - error = turn.get("error") or {} - return [ - Event( - invocation_id=invocation_id, - author=author, - turn_complete=True, - error_code=str(error.get("code") or "codex_error") if error else None, - error_message=str(error.get("message") or error) if error else None, - custom_metadata={ - "codex_event_type": "turn_complete", - "turn_id": turn.get("id"), - "status": _scalar(turn.get("status")) or "completed", - }, - ) - ] - return [] +def _on_context_compacted( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Report that Codex compacted the thread's history mid-turn. + + Compaction silently drops earlier context and so changes what the model can + still see. Surfacing it gives Trace/UI consumers a marker for an otherwise + invisible discontinuity. + """ + return [ + _lifecycle_event( + author, + invocation_id, + "context_compacted", + { + "thread_id": data.get("thread_id"), + "turn_id": data.get("turn_id"), + }, + ) + ] + + +def _on_model_rerouted( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Report that Codex served the turn with a model we did not request.""" + return [ + _lifecycle_event( + author, + invocation_id, + "model_rerouted", + { + "turn_id": data.get("turn_id"), + "from_model": data.get("from_model"), + "to_model": data.get("to_model"), + "reason": _scalar(data.get("reason")), + }, + ) + ] + + +def _on_error( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Translate a turn-scoped error into an ADK error event. + + A retryable error is a progress signal, so it is marked ``partial`` and + stays out of session history; a terminal one has to persist. + """ + error = data.get("error") or {} + will_retry = bool(data.get("will_retry")) + return [ + Event( + invocation_id=invocation_id, + author=author, + partial=True if will_retry else None, + error_code=_error_code(error), + error_message=str(error.get("message") or error), + custom_metadata={ + "codex_event_type": "error", + "will_retry": will_retry, + }, + ) + ] + + +def _on_turn_completed( + data: dict[str, Any], author: str, invocation_id: str, active_tool_items: set[str] +) -> list[Event]: + """Close the turn, propagating ``turn.error`` when the turn failed. + + A clean completion is a contentless control marker, so it is ``partial`` + and never reads as the agent's final answer; a failed turn carries a real + error and has to persist. + """ + turn = data.get("turn") or {} + error = turn.get("error") or {} + return [ + Event( + invocation_id=invocation_id, + author=author, + turn_complete=True, + partial=None if error else True, + error_code=_error_code(error) if error else None, + error_message=str(error.get("message") or error) if error else None, + custom_metadata={ + "codex_event_type": "turn_complete", + "turn_id": turn.get("id"), + "status": _scalar(turn.get("status")) or "completed", + }, + ) + ] + + +# Turn-scoped notifications that deliberately translate to no ADK event. +# Ignoring them is a recorded decision, not an oversight: the coverage test +# asserts ``set(_DISPATCH) | _EXPLICITLY_IGNORED`` equals the SDK's full +# turn-scoped notification set in both directions, so a new or renamed SDK type +# fails the suite instead of being silently dropped here. +_EXPLICITLY_IGNORED: frozenset[str] = frozenset( + { + # Structural marker only (item id + summary index). The reasoning text + # itself arrives on ReasoningSummaryTextDeltaNotification. + "ReasoningSummaryPartAddedNotification", + # Cumulative unified diff for the whole turn, resent on every edit. + # Per-file changes already reach ADK through + # FileChangePatchUpdatedNotification and the completed fileChange item. + "TurnDiffUpdatedNotification", + # Raw stdin written into an interactive terminal session. Replaying it + # would duplicate the command's own output and can echo typed secrets. + "TerminalInteractionNotification", + # Locally configured hook runs: operator tooling around the turn rather + # than model or tool output. + "HookStartedNotification", + "HookCompletedNotification", + # Thread-scoped goal bookkeeping (its turn_id is optional); not a + # product of this turn. + "ThreadGoalUpdatedNotification", + # Account-level attestation notice (e.g. "trustedAccessForCyber") with + # no per-turn meaning. + "ModelVerificationNotification", + } +) + +# Notification class name -> handler. Keyed on ``type(payload).__name__`` so +# this module never has to import the optional ``openai_codex`` package. +_DISPATCH: dict[str, _NotificationHandler] = { + "AgentMessageDeltaNotification": _delta_handler("message_delta"), + "CommandExecutionOutputDeltaNotification": _delta_handler("command_output"), + "ContextCompactedNotification": _on_context_compacted, + "ErrorNotification": _on_error, + "FileChangeOutputDeltaNotification": _delta_handler("file_change_output"), + "FileChangePatchUpdatedNotification": _on_file_change_patch, + "ItemCompletedNotification": _on_item_completed, + "ItemGuardianApprovalReviewCompletedNotification": _approval_handler("completed"), + "ItemGuardianApprovalReviewStartedNotification": _approval_handler("in_progress"), + "ItemStartedNotification": _on_item_started, + "McpToolCallProgressNotification": _delta_handler("mcp_progress"), + "ModelReroutedNotification": _on_model_rerouted, + "PlanDeltaNotification": _delta_handler("plan_delta"), + "ReasoningSummaryTextDeltaNotification": _delta_handler( + "reasoning_delta", thought=True + ), + "ReasoningTextDeltaNotification": _delta_handler("reasoning_delta", thought=True), + "ThreadTokenUsageUpdatedNotification": _on_token_usage, + "TurnCompletedNotification": _on_turn_completed, + "TurnPlanUpdatedNotification": _on_turn_plan_updated, + "TurnStartedNotification": _on_turn_started, +} + + +def notification_to_events( + payload: Any, + author: str, + invocation_id: str, + *, + active_tool_items: set[str] | None = None, +) -> list[Event]: + """Translate a Codex lifecycle notification into observable ADK events. + + Completed items still use :func:`item_to_events`, while starts, output + deltas, plan changes, approval reviews, context compaction, model reroutes, + turn completion, and errors carry a stable + ``custom_metadata.codex_event_type`` for Trace/UI consumers. + + Dispatch is a table lookup on ``type(payload).__name__`` (:data:`_DISPATCH` + plus :data:`_EXPLICITLY_IGNORED`), which keeps the module importable + without the optional ``openai_codex`` extra while making the set of + unhandled SDK types enumerable by tests instead of silently dropped. + + Args: + payload (Any): A Codex notification payload (model or dict). + author (str): Event author (the agent name). + invocation_id (str): The ADK invocation id to stamp on each event. + active_tool_items (set[str] | None): Ids of tool items whose + ``function_call`` part was already emitted at item start. + + Returns: + list[google.adk.events.event.Event]: Events for this notification; + empty when it carries nothing observable. + """ + handler = _DISPATCH.get(type(payload).__name__) + if handler is None: + return [] + return handler( + _item_dict(payload), + author, + invocation_id, + active_tool_items if active_tool_items is not None else set(), + ) def _lifecycle_event( @@ -681,8 +1039,18 @@ def _lifecycle_event( metadata: dict[str, Any], *, part: types.Part | None = None, - partial: bool | None = None, + partial: bool | None = True, ) -> Event: + """Build one observable, non-final Codex lifecycle event. + + ``partial`` defaults to ``True`` on purpose. ``Event.is_final_response()`` + is true for any contentless, tool-free, non-partial event, so a plain + lifecycle marker would otherwise read as the agent's final answer. These + events are also write-only in history -- ``build_prompt`` and the model + callbacks both drop parts-less records -- so keeping them out of the + session costs nothing. Pass ``partial=None`` for the rare marker that has + to persist. + """ return Event( invocation_id=invocation_id, author=author, diff --git a/veadk/runtime/codex/workspace.py b/veadk/runtime/codex/workspace.py new file mode 100644 index 000000000..ab95bc048 --- /dev/null +++ b/veadk/runtime/codex/workspace.py @@ -0,0 +1,165 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The running Codex turn's workspace, as an ADK tool sees it. + +Why a tool needs this +--------------------- + +Under ``runtime="codex"`` the two halves of a turn run in different places. +Codex runs in a sandboxed subprocess whose ``cwd`` is a per-session workspace +directory; the agent's ADK tools run in the host process, outside that sandbox. +So the supported way for a tool to hand Codex data is to **write a file into +that workspace and return its path** — never to return the payload, which the +shim serializes into the model's context and replays on every later request of +the turn. + +That leaves the tool needing the path, and it cannot derive one: the workspace +is keyed by a private digest over app/user/session/agent (see +:func:`veadk.runtime.codex.runtime._prepare_workspace`). Pinning +``workspace_root`` together with ``reuse_workspace`` makes it predictable, but +collapses every session onto one directory — acceptable for a single-user demo, +not for a multi-tenant server. :func:`current_workspace` is the supported +alternative, and it works with both left unset. + +Why the value is bound at *tool-call* time +------------------------------------------ + +The obvious shape — set a ``ContextVar`` in ``run_async`` and read it in the +tool — does not work, and fails in the worst possible way. ADK tools are run by +the Responses shim, in a request handler that descends from the uvicorn server +task, and that task's context was snapshotted by ``asyncio.create_task`` when +the *first* invocation in the process started the shim. Measured against a real +shim (var set in the invocation's own task, three later turns on the same +shim): every later turn's tool read the **first** turn's value. A plain +"contextvar set in ``run_async``" is therefore not a miss but a silent +cross-tenant leak. The same asymmetry is why the shim has to capture an OTel +context at ``register_turn`` and re-attach it around tool execution; see +``proxy.ShimTurnContext.otel_context``. + +Nothing here reads ambient context to *find* the workspace, then. The runtime +wraps each turn's executors with :func:`bind_workspace_to_executors`, which +captures the workspace in a closure, and every wrapper sets the ``ContextVar`` +around its own call. Whichever task the shim runs an executor on, the value is +set in *that* task's context — ``asyncio.gather`` gives each concurrent call its +own context copy — so a tool can read it ambiently while the binding itself can +never be inherited by the wrong turn. +""" + +from __future__ import annotations + +import contextlib +import contextvars +from typing import Any, Awaitable, Callable, Iterator + +#: ``name -> async (args, call_id) -> str``, as built by +#: :mod:`veadk.runtime.codex.tools_bridge` and consumed by the shim. +Executor = Callable[[dict[str, Any], str], Awaitable[str]] + +_CURRENT_WORKSPACE: contextvars.ContextVar[str | None] = contextvars.ContextVar( + "veadk_codex_current_workspace", default=None +) + + +def current_workspace() -> str | None: + """Absolute path of the workspace Codex is using for the current tool call. + + Call this from an ADK tool running under ``runtime="codex"`` to find the + directory the sandbox is working in, then write files there and return + their paths to the model instead of their contents. + + Returns ``None`` (rather than raising) when there is no Codex turn on this + call stack — the same tool object is routinely executed by other runtimes, + by ``AgentTool``, and by unit tests, and a tool that can branch on ``None`` + stays usable in all of them. A tool that genuinely cannot work without the + workspace should return its own ``{"status": "error", ...}`` result, which + the model can act on, rather than raising out of the tool. + + Returns: + str | None: The turn's workspace directory, or ``None`` outside a + Codex tool call. + + Example: + :: + + from pathlib import Path + + from veadk.runtime.codex import current_workspace + + + def fetch_orders(quarter: str) -> dict: + \"\"\"Write a quarter of orders into your working directory.\"\"\" + workspace = current_workspace() + if workspace is None: + return {"status": "error", "message": "no codex workspace"} + destination = Path(workspace) / "data" / f"{quarter}.csv" + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(export_orders(quarter), encoding="utf-8") + # A receipt, not the rows: the rows would be replayed into the + # model's context on every later request of this turn. + return {"status": "ok", "path": f"data/{quarter}.csv"} + """ + return _CURRENT_WORKSPACE.get() + + +@contextlib.contextmanager +def bind_workspace(workspace: str) -> Iterator[None]: + """Make ``workspace`` the value :func:`current_workspace` returns. + + Scoped to the block *and* to the task running it: a ``ContextVar`` set here + is visible to everything this task awaits (including tasks it spawns, which + copy the context at creation) and to nothing else. + + Args: + workspace (str): Absolute path of the turn's workspace. + + Yields: + None: For the duration of the binding. + """ + token = _CURRENT_WORKSPACE.set(workspace) + try: + yield + finally: + _CURRENT_WORKSPACE.reset(token) + + +def bind_workspace_to_executors( + executors: dict[str, Executor], workspace: str +) -> dict[str, Executor]: + """Wrap tool executors so each call runs with ``workspace`` bound. + + The workspace is captured in the wrapper's closure rather than read from + ambient context, which is what makes this correct wherever the shim decides + to run the executor: the shim's handler task does not inherit the + invocation's context (see the module docstring), so a value the invocation + merely *set* would not be there — or, worse, a value another invocation set + before the shim started would be. + + Args: + executors (dict[str, Executor]): The turn's tool executors. + workspace (str): Absolute path of the turn's workspace. + + Returns: + dict[str, Executor]: Executors of the same shape, each binding the + workspace for the duration of its own call. + """ + + def _bind(executor: Executor) -> Executor: + async def _run(args: dict[str, Any], call_id: str) -> str: + with bind_workspace(workspace): + return await executor(args, call_id) + + return _run + + return {name: _bind(executor) for name, executor in executors.items()} diff --git a/veadk/runtime/compat.py b/veadk/runtime/compat.py new file mode 100644 index 000000000..faa23b3f0 --- /dev/null +++ b/veadk/runtime/compat.py @@ -0,0 +1,487 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Support matrix for non-``adk`` agent runtimes. + +``Agent(runtime="codex")`` and ``Agent(runtime="piagent")`` replace the *entire* +ADK LLM flow with an external harness. Everything ADK implements inside that +flow — agent transfer, request/response processors, the ``LlmRequest`` +assembly, per-call callbacks — therefore does not run. Without a check, a large +part of :class:`veadk.agent.Agent`'s configuration surface is accepted at +construction time and then silently ignored at run time. + +This module is the single, runtime-agnostic place that states which +configuration is unsupported and how loudly to say so: + +* ``"error"`` — the configuration produces a *wrong answer* (not just a missing + feature), so the agent must not run at all. Raised as :class:`ValueError`. +* ``"warn"`` — the configuration is dropped but the turn still produces a + reasonable answer. Logged once per ``(agent identity, field)`` so a + per-request cloned agent does not spam the log. + +Rules read agent state defensively, so a duck-typed stand-in that happens not to +carry a field simply fires no rule. That keeps the checker usable from +:meth:`veadk.agent.Agent._run_async_impl` without constraining what the runtime +tests may pass directly into ``Runtime.run_async``. + +Deliberately *not* covered: ``context_cache_config`` and ``parallel_worker``. +Neither is read by veadk or google-adk 2.2.0 on the ``adk`` path either, so +blaming the runtime for ignoring them would be wrong. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field as dataclass_field +from typing import Any, Callable, Literal, Optional + +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) + +Policy = Literal["error", "warn"] +"""How loudly an unsupported configuration is reported.""" + +ALL_RUNTIMES: frozenset[str] = frozenset({"codex", "piagent"}) +"""Every runtime that bypasses ADK's LLM flow.""" + +EXPLICIT_FIELDS_ATTR = "_veadk_explicit_fields" +"""Private attribute holding the caller-set field names. + +:class:`veadk.agent.Agent` snapshots ``model_fields_set`` into this attribute at +the *top* of ``model_post_init``. The raw ``model_fields_set`` cannot be used: +``Agent.model_post_init`` assigns ``model``/``model_extra_config``/ +``run_processor`` itself, and ``BaseAgent.clone()`` re-assigns every list field, +so by the time anything can be checked those names are always "set". +""" + + +def explicit_fields(agent: Any) -> frozenset[str]: + """Return the field names the *caller* passed to the agent constructor. + + Args: + agent (Any): The agent (or duck-typed stand-in) to inspect. + + Returns: + frozenset[str]: Caller-set field names. Falls back to + ``agent.model_fields_set`` when no snapshot is available, and to an + empty set for objects that are not pydantic models. + """ + snapshot = getattr(agent, EXPLICIT_FIELDS_ATTR, None) + if isinstance(snapshot, (frozenset, set)): + return frozenset(snapshot) + return frozenset(getattr(agent, "model_fields_set", frozenset()) or ()) + + +@dataclass(frozen=True) +class SupportRule: + """One unsupported-configuration rule. + + Attributes: + field (str): The ``Agent`` field the rule is about. Also the dedup key + for ``warn`` rules. + policy (Policy): ``"error"`` or ``"warn"``. + predicate (Callable[[Any, frozenset[str]], bool]): Receives the agent + and its :func:`explicit_fields`; returns whether the rule fires. + message (Callable[[Any, str], str]): Receives the agent and the runtime + name; returns the user-facing message. + applies_to (frozenset[str]): Runtimes the rule applies to. + """ + + field: str + policy: Policy + predicate: Callable[[Any, frozenset[str]], bool] + message: Callable[[Any, str], str] + applies_to: frozenset[str] = dataclass_field(default=ALL_RUNTIMES) + + +def _truthy(name: str) -> Callable[[Any, frozenset[str]], bool]: + """Build a predicate that fires when ``agent.`` is truthy.""" + + def _predicate(agent: Any, _explicit: frozenset[str]) -> bool: + return bool(getattr(agent, name, None)) + + return _predicate + + +def _explicitly_set(name: str) -> Callable[[Any, frozenset[str]], bool]: + """Build a predicate that fires when the caller passed ``name``.""" + + def _predicate(_agent: Any, explicit: frozenset[str]) -> bool: + return name in explicit + + return _predicate + + +def _dropped_generate_content_fields(agent: Any) -> list[str]: + """Return ``generate_content_config`` fields no external runtime forwards. + + Only ``system_instruction`` survives the bridge into an external harness; + everything else (``temperature``, ``max_output_tokens``, ``thinking_config``, + ...) is assembled by ADK's flow and never reaches the backend here. + """ + config = getattr(agent, "generate_content_config", None) + if config is None: + return [] + fields = getattr(config, "model_fields_set", None) + if not fields: + return [] + return sorted(set(fields) - {"system_instruction"}) + + +def _model_name_fallbacks(agent: Any) -> list[str]: + """Return the ``model_name`` entries after the first one.""" + model_name = getattr(agent, "model_name", None) + if isinstance(model_name, list) and len(model_name) > 1: + return [str(name) for name in model_name[1:]] + return [] + + +SUPPORT_RULES: tuple[SupportRule, ...] = ( + # --- error: silently wrong results ------------------------------------- + SupportRule( + field="model", + policy="error", + predicate=_explicitly_set("model"), + message=lambda _agent, rt: ( + f"{rt} runtime resolves the model from Agent(model_name=...) and " + "ignores the Agent(model=...) object entirely, so its api_base, " + "headers and fallbacks are dropped. Set model_name instead, or use " + "runtime='adk'." + ), + ), + SupportRule( + field="generate_content_config", + policy="error", + predicate=lambda agent, _explicit: bool( + _dropped_generate_content_fields(agent) + ), + message=lambda agent, rt: ( + f"{rt} runtime forwards only " + "generate_content_config.system_instruction; " + f"{', '.join(_dropped_generate_content_fields(agent))} would be " + "silently dropped. Remove them, or use runtime='adk'." + ), + ), + SupportRule( + field="output_schema", + policy="error", + predicate=_truthy("output_schema"), + message=lambda _agent, rt: ( + f"{rt} runtime sends Agent(output_schema=...) neither to the " + "backend nor into the prompt, so the model is never asked for the " + "schema and state[output_key] would hold an unvalidated reply or " + "silently be missing. Use runtime='adk', or drop output_schema and " + "parse the reply yourself." + ), + ), + SupportRule( + field="planner", + policy="error", + predicate=_truthy("planner"), + message=lambda _agent, rt: ( + "Agent(planner=...) runs as an ADK request/response processor, " + f"which {rt} runtime never executes, so it has no effect on the " + "turn. Use runtime='adk', or remove planner." + ), + ), + SupportRule( + field="code_executor", + policy="error", + predicate=_truthy("code_executor"), + message=lambda _agent, rt: ( + "Agent(code_executor=...) runs as an ADK request/response " + f"processor, which {rt} runtime never executes, so it has no effect " + "on the turn. Use runtime='adk', or remove code_executor." + ), + ), + SupportRule( + field="include_contents", + policy="error", + predicate=lambda agent, _explicit: ( + getattr(agent, "include_contents", None) == "none" + ), + message=lambda _agent, rt: ( + f"{rt} runtime always sends the full conversation history; " + "include_contents='none' would be silently ignored and prior turns " + "leaked to the model. Use runtime='adk', or leave include_contents " + "at 'default'." + ), + ), + SupportRule( + field="enable_supervisor", + policy="error", + predicate=_truthy("enable_supervisor"), + message=lambda _agent, rt: ( + "Agent(enable_supervisor=True) is installed through the ADK LLM " + f"flow, which {rt} runtime replaces, so no supervision runs. Use " + "runtime='adk', or set enable_supervisor=False." + ), + ), + # --- warn: dropped, but the turn still answers -------------------------- + SupportRule( + field="model_name", + policy="warn", + predicate=lambda agent, _explicit: bool(_model_name_fallbacks(agent)), + message=lambda agent, rt: ( + f"{rt} runtime uses only the first entry of Agent(model_name=[...]) " + f"and drops the fallbacks {_model_name_fallbacks(agent)}, because " + "the fallback chain lives on the LiteLLM client this runtime never " + "builds; a backend failure will surface as an error instead of " + "failing over. Pass a single model name, or use runtime='adk' if " + "you need fallbacks." + ), + ), + SupportRule( + field="model_provider", + policy="warn", + predicate=lambda agent, _explicit: bool(getattr(agent, "model_provider", None)) + and getattr(agent, "model_provider", None) != "openai", + message=lambda agent, rt: ( + f"{rt} runtime always talks to model_api_base over an " + "OpenAI-compatible API and ignores " + f"Agent(model_provider={getattr(agent, 'model_provider', None)!r}), " + "because the provider prefix only selects a LiteLLM client that is " + "never created here; a non-OpenAI-compatible endpoint will fail at " + "call time. Use runtime='adk' if the provider matters." + ), + ), + SupportRule( + field="model_extra_config", + policy="warn", + predicate=_explicitly_set("model_extra_config"), + message=lambda _agent, rt: ( + f"{rt} runtime drops Agent(model_extra_config=...) — both " + "extra_headers and extra_body, including the VeADK Ark defaults for " + "request encryption and prompt caching — because it does not build " + "the LiteLLM/Ark client those options configure, so requests go out " + "unencrypted and uncached. Use runtime='adk' if you need them." + ), + # codex forwards it: the runtime hands it to `register_turn` and the + # shim applies extra_headers/extra_body to every backend call. + applies_to=frozenset({"piagent"}), + ), + SupportRule( + field="enable_responses", + policy="warn", + predicate=_truthy("enable_responses"), + message=lambda _agent, rt: ( + f"{rt} runtime drives the backend itself and ignores " + "Agent(enable_responses=True) together with enable_responses_cache, " + "because the Ark Responses client (and its previous_response_id " + "continuation and response caching) is part of the ADK model layer " + "this runtime replaces. Use runtime='adk' to get the Responses API." + ), + ), + SupportRule( + field="example_store", + policy="warn", + predicate=_truthy("example_store"), + message=lambda _agent, rt: ( + "Agent(example_store=...) is delivered by " + f"ExampleTool.process_llm_request, which {rt} runtime never calls, " + "so no few-shot examples reach the model. Use runtime='adk', or " + "put the examples in the instruction." + ), + ), + SupportRule( + field="knowledgebase", + policy="warn", + predicate=_truthy("knowledgebase"), + message=lambda _agent, rt: ( + "Agent(knowledgebase=...) is silently disabled under " + f"{rt} runtime: the VeADK knowledge base is wired in by " + "LoadKnowledgebaseTool.process_llm_request, the ADK hook that tells " + f"the model the knowledge base exists and when to query it, and {rt} " + "runtime never calls it, so retrieval is not triggered and answers " + "fall back to the model's own knowledge. Use runtime='adk', or " + "perform the retrieval yourself and pass the result in the " + "instruction." + ), + ), + SupportRule( + field="skills_mode", + policy="warn", + predicate=_truthy("skills_mode"), + message=lambda agent, rt: ( + "Agent(skills_mode=" + f"{getattr(agent, 'skills_mode', None)!r}) has no effect: {rt} " + "runtime skips VeADK's SkillsToolset when bridging tools, so the " + "instruction advertises execute_skills/skills_tool while no such " + "tool is registered and every call fails. Use runtime='adk', or " + "pass the skills through a runtime-native skill toolset instead." + ), + ), + SupportRule( + field="enable_skills_checklist", + policy="warn", + predicate=_truthy("enable_skills_checklist"), + message=lambda _agent, rt: ( + "Agent(enable_skills_checklist=True) installs an ADK before-tool " + f"callback over VeADK skills, and {rt} runtime bridges neither the " + "skills toolset nor per-tool ADK callbacks, so no checklist is ever " + "enforced. Use runtime='adk', or set enable_skills_checklist=False." + ), + ), + SupportRule( + field="after_model_callback", + policy="warn", + predicate=_truthy("after_model_callback"), + message=lambda _agent, rt: ( + f"Agent(after_model_callback=...) behaves differently under {rt} " + "runtime: it fires once per turn on the merged final text, not once " + "per LLM call, because the inner model loop runs inside the " + "external harness, so intermediate model responses cannot be " + "inspected or rewritten. Use runtime='adk' if you need per-call " + "callbacks." + ), + ), + SupportRule( + field="tracers", + policy="warn", + predicate=_truthy("tracers"), + # Deliberately does not claim a per-turn span exists: whether one is + # emitted is per-runtime, but "no span per model call" holds for both. + message=lambda _agent, rt: ( + f"Agent(tracers=...) records no per-model-call spans under {rt} " + "runtime: the harness's own loop issues several backend calls per " + "turn and is not instrumented by ADK, so their individual prompts, " + "responses and token splits are not broken out. Use runtime='adk' " + "for per-call tracing." + ), + ), + SupportRule( + field="codex_runtime_config", + policy="warn", + predicate=_truthy("codex_runtime_config"), + applies_to=frozenset({"piagent"}), + message=lambda agent, rt: ( + "Agent(codex_runtime_config=...) configures the codex runtime only; " + f"{rt} runtime ignores it, so its sandbox, approval_mode, " + "network_access and workspace settings do not apply to this agent. " + "Remove it, or set runtime='codex'." + ), + ), +) + + +_RUN_CONFIG_FIELD = "run_config.max_llm_calls" + + +def _run_config_message(runtime: str) -> str: + # ADK's only enforcement point is + # ``InvocationContext.increment_llm_call_count()``, reached from + # ``base_llm_flow``, which no external runtime executes. codex charges the + # budget itself before each backend call, so it never reaches this message + # (see the guard in ``check_agent_runtime_support``). piagent can only + # charge a call the Pi binary has already finished, hence the overshoot. + return ( + f"RunConfig(max_llm_calls=...) is enforced one call late under " + f"{runtime} runtime: the model loop runs inside the external harness, " + "which reports a call only once it has completed, so the invocation " + "aborts just past the limit rather than just short of it. Use " + "runtime='adk' for exact enforcement." + ) + + +_WARNED: set[tuple[str, str]] = set() +"""``(agent identity, field)`` pairs already warned about in this process.""" + + +def _agent_identity(agent: Any) -> str: + """Return a stable identity for warning deduplication. + + ``Agent.id`` is preferred so that per-request clones (which copy ``id``) + share one warning instead of logging on every request. + """ + identity = getattr(agent, "id", None) + if isinstance(identity, str) and identity: + return identity + return f"{type(agent).__name__}:{id(agent):x}" + + +def _warn_once(agent: Any, field: str, message: str) -> None: + key = (_agent_identity(agent), field) + if key in _WARNED: + return + _WARNED.add(key) + logger.warning(message) + + +def reset_warning_state() -> None: + """Clear the once-per-``(agent, field)`` warning cache. For tests.""" + _WARNED.clear() + + +def check_agent_runtime_support( + agent: Any, + runtime: str, + *, + run_config: Optional[Any] = None, +) -> None: + """Validate an agent against a runtime's support matrix. + + Args: + agent (Any): The agent about to run. Read defensively, so a duck-typed + stand-in without the inspected fields fires no rule. + runtime (str): Runtime name from ``Agent(runtime=...)``. ``"adk"`` (and + any falsy value) returns immediately. + run_config (Optional[Any]): The invocation's ``RunConfig``, when the + check runs at invocation time. Fields that only exist per + invocation are checked only when this is provided. + + Raises: + ValueError: On the first ``error``-policy violation, in the declaration + order of :data:`SUPPORT_RULES`. ``warn``-policy violations are + logged once per ``(agent identity, field)`` and never raise. + """ + if not runtime or runtime == "adk": + return + + explicit = explicit_fields(agent) + for rule in SUPPORT_RULES: + if runtime not in rule.applies_to: + continue + try: + triggered = rule.predicate(agent, explicit) + except Exception: # noqa: BLE001 - a probe must never break a run + continue + if not triggered: + continue + if rule.policy == "error": + raise ValueError(rule.message(agent, runtime)) + _warn_once(agent, rule.field, rule.message(agent, runtime)) + + # codex charges every backend model *call* through + # `ResponsesShim.register_turn(on_model_call=...)`, *before* the call, so + # the budget binds exactly and warning would be a false positive. litellm's + # `num_retries` re-attempts a failed call underneath that charge, the same + # way it does on the adk path, so a call is counted once either way. piagent + # charges each call the Pi binary reports as finished, which still enforces + # the budget but only after the overrunning call has run. + if run_config is not None and runtime != "codex": + fields = getattr(run_config, "model_fields_set", None) or () + if "max_llm_calls" in fields: + _warn_once(agent, _RUN_CONFIG_FIELD, _run_config_message(runtime)) + + +__all__ = [ + "ALL_RUNTIMES", + "EXPLICIT_FIELDS_ATTR", + "Policy", + "SUPPORT_RULES", + "SupportRule", + "check_agent_runtime_support", + "explicit_fields", + "reset_warning_state", +] diff --git a/veadk/runtime/model_callbacks.py b/veadk/runtime/model_callbacks.py index cf6505b52..ee265161d 100644 --- a/veadk/runtime/model_callbacks.py +++ b/veadk/runtime/model_callbacks.py @@ -374,3 +374,49 @@ def _same_content(left: Any, right: Any) -> bool: mode="json", exclude_none=True ) return left == right + + +#: Action fields carried from a dropped merged event onto the last emitted one. +#: `skip_summarization` is deliberately excluded: `Event.is_final_response()` +#: returns True for any event that sets it, so copying it would turn the tool +#: event into the invocation's final response - exactly what withholding the +#: contentless merged event is meant to prevent. +_MERGED_ACTION_DICTS = ( + "state_delta", + "artifact_delta", + "requested_auth_configs", + "requested_tool_confirmations", +) +_MERGED_ACTION_SCALARS = ("transfer_to_agent", "escalate", "end_invocation") + + +def merge_turn_bookkeeping(target: Event, merged: Event) -> None: + """Fold a tool-only turn's merged-response bookkeeping onto ``target``. + + This module owns both producers of that bookkeeping: + ``run_before_model_callbacks``/``run_after_model_callbacks`` build their + ``CallbackContext`` over ``model_response_event.actions``, so a callback's + ``callback_context.state[...]`` writes land on the merged event's + ``state_delta``; ``llm_response_to_event`` then also attaches the turn's + ``usage_metadata``. When the merged event has no content it cannot be + emitted (it would read as the final response), so that bookkeeping is moved + onto the last event this turn actually emits. + + Args: + target (Event): The already-built event that will be emitted. + merged (Event): The contentless merged response being withheld. + """ + source = getattr(merged, "actions", None) + destination = getattr(target, "actions", None) + if source is not None and destination is not None: + for name in _MERGED_ACTION_DICTS: + values = getattr(source, name, None) + if values: + getattr(destination, name).update(values) + for name in _MERGED_ACTION_SCALARS: + value = getattr(source, name, None) + if value: + setattr(destination, name, value) + usage = getattr(merged, "usage_metadata", None) + if usage is not None and getattr(target, "usage_metadata", None) is None: + target.usage_metadata = usage diff --git a/veadk/runtime/output_state.py b/veadk/runtime/output_state.py index 9cf8eb3c2..fe7e691a2 100644 --- a/veadk/runtime/output_state.py +++ b/veadk/runtime/output_state.py @@ -16,22 +16,60 @@ from __future__ import annotations -from typing import Any +from typing import Any, Optional from google.adk.events.event import Event from google.adk.utils._schema_utils import validate_schema +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) + def maybe_save_output_to_state(agent: Any, event: Event) -> None: """Save a final model text response to ``event.actions.state_delta``. This mirrors ADK's ``LlmAgent.__maybe_save_output_to_state`` for runtimes - that bypass ADK's built-in LLM flow. + that bypass ADK's built-in LLM flow, with two behaviours that ADK does not + need but external harnesses do. + + **Last write wins.** ``Event.is_final_response()`` is True for *every* + non-partial, tool-free text event, so a harness that streams several + complete assistant messages in one turn (Codex ``agentMessage`` items) makes + this run once per message. Each run overwrites ``state_delta[output_key]``, + so the value that survives the turn is the agent's *last* message — never a + concatenation of its intermediate thinking. That is the ADK meaning of + ``output_key`` ("the agent's answer") and it is what makes ``output_schema`` + validation meaningful. It is also stable if a runtime later marks + intermediate items ``partial=True``: those stop being final responses, only + the real final message reaches this function, and the outcome is unchanged. + + **The ``output_schema`` branch is unreachable by construction.** + ``veadk.runtime.compat`` classifies ``output_schema`` as ``"error"`` for + every non-``adk`` runtime, and that gate runs a few lines above this + function's only call site in :meth:`veadk.agent.Agent._run_async_impl`. The + rule is right: the schema reaches neither the backend nor the prompt (it is + written to ``LlmRequest.config.response_schema``, which the prompt builder + never reads), so the model is not constrained and "structured output" would + not be structured. + + The branch is nonetheless *guarded rather than raising*, so that demoting + the rule can never kill a turn mid-stream: ``validate_schema`` calls + ``model_validate_json``, which raises on prose, and this runs on every + event of the invocation. A non-conforming final text is skipped with a + warning; any value an earlier event wrote stays in place. The real + protection against an accidental demotion is the test asserting that rule + stays at ``"error"`` — see ``tests/runtime/differential/``. + + Args: + agent (Any): The agent that produced the event. Read defensively. + event (google.adk.events.event.Event): The event to inspect. Mutated in + place when a value is saved. """ if event.author != agent.name: return - output_key = getattr(agent, "output_key", None) + output_key: Optional[str] = getattr(agent, "output_key", None) if not output_key: return @@ -48,7 +86,7 @@ def maybe_save_output_to_state(agent: Any, event: Event) -> None: if not has_text_part: return - result = "".join( + result: Any = "".join( part.text for part in event.content.parts if part.text and not getattr(part, "thought", False) @@ -56,8 +94,21 @@ def maybe_save_output_to_state(agent: Any, event: Event) -> None: output_schema = getattr(agent, "output_schema", None) if output_schema: + # Unreachable while compat keeps output_schema at "error" — see the + # docstring. Guarded anyway so a demotion cannot kill a turn mid-stream. if not result.strip(): return - result = validate_schema(output_schema, result) + try: + result = validate_schema(output_schema, result) + except Exception: # noqa: BLE001 - pydantic/json errors must not kill a run + logger.warning( + "Agent '%s' final response does not match output_schema %s; " + "skipping the state['%s'] write. Drop output_schema, or use " + "runtime='adk'.", + getattr(agent, "name", ""), + getattr(output_schema, "__name__", type(output_schema).__name__), + output_key, + ) + return event.actions.state_delta[output_key] = result diff --git a/veadk/runtime/piagent/runtime.py b/veadk/runtime/piagent/runtime.py index b65a41f5c..13a866143 100644 --- a/veadk/runtime/piagent/runtime.py +++ b/veadk/runtime/piagent/runtime.py @@ -20,6 +20,8 @@ from collections.abc import AsyncGenerator from typing import TYPE_CHECKING +from google.adk.agents.invocation_context import LlmCallsLimitExceededError + from veadk.runtime.agent_transfer import ( append_transfer_instructions, build_transfer_tool, @@ -29,9 +31,10 @@ ) from veadk.runtime.base_runtime import BaseRuntime from veadk.runtime.model_callbacks import ( + merge_turn_bookkeeping, + RuntimeLlmCall, build_runtime_llm_request, final_events_to_llm_response, - has_after_model_callbacks, is_final_model_text_event, llm_response_to_event, run_after_model_callbacks, @@ -52,12 +55,15 @@ from veadk.runtime.piagent.translate import ( PiEventTranslator, build_prompt_from_llm_request, + counts_as_model_call, ) from veadk.utils.logger import get_logger if TYPE_CHECKING: from google.adk.agents.invocation_context import InvocationContext from google.adk.events.event import Event + from google.adk.models.llm_response import LlmResponse + from opentelemetry.trace import Span from veadk.agent import Agent @@ -83,6 +89,11 @@ async def run_async( async def _emit_tool_event(event: Event) -> None: await event_queue.put(event) + # ADK's `call_llm` span is opened by its own LLM flow, which this + # runtime replaces. Open it here so VeADK's telemetry chain (the + # in-memory exporter's session index, the evaluator, portal metrics and + # the common model-span attributes) sees a Pi invocation at all. + call_llm_span = _start_call_llm_span() try: tool_bundle = await build_executable_tools( agent, ctx, event_sink=_emit_tool_event @@ -110,6 +121,9 @@ async def _emit_tool_event(event: Event) -> None: runtime_call.model_response_event, ) if short_circuit is not None: + _emit_call_llm_telemetry( + ctx, runtime_call, short_circuit, call_llm_span + ) yield llm_response_to_event( runtime_call.llm_request, short_circuit, @@ -140,8 +154,20 @@ async def _emit_tool_event(event: Event) -> None: invocation_id=ctx.invocation_id, bridged_tool_names=set(tool_bundle.executors), ) - buffer_final_text = has_after_model_callbacks(agent, ctx) + # Buffered unconditionally: gating this on a registered after-model + # callback made otherwise identical agents produce different event + # streams, and let an intermediate assistant message read as the + # turn's final response. final_text_events: list[Event] = [] + # Lookahead for the tool-only turn. That turn's merged response + # carries the turn's `usage_metadata` and any `state_delta` a model + # callback wrote, but it has no content, and a contentless, + # tool-free, non-partial event reads as the invocation's final + # response (`Event.is_final_response()`) -- so it cannot simply be + # emitted. The last *durable* event is therefore held back to give + # that bookkeeping somewhere real to land; see + # `_merge_turn_bookkeeping`. + merge_target: "Event | None" = None async with PiToolRuntime(tool_bundle) as tools: run_config = ( config.with_skills(skill_paths=list(skill_bundle.paths)) @@ -154,18 +180,20 @@ async def _emit_tool_event(event: Event) -> None: else run_config ) async with PiAgentRpcClient(run_config) as client: - pump: asyncio.Task[None] | None = None async def _pump_pi() -> None: try: async for pi_event in client.prompt(prompt): + if counts_as_model_call(pi_event): + _charge_llm_call(ctx) for event in translator.event_to_adk_events(pi_event): await event_queue.put(event) - except Exception as e: # noqa: BLE001 - forward pump errors + except BaseException as e: # noqa: BLE001 - forward pump errors await event_queue.put(e) finally: await event_queue.put(_QUEUE_DONE) + pump: asyncio.Task[None] | None = None try: pump = asyncio.create_task(_pump_pi()) while True: @@ -175,15 +203,30 @@ async def _pump_pi() -> None: if isinstance(queued, BaseException): raise queued event = queued # type: ignore[assignment] - transfer_target = transfer_agent_name(event) - if buffer_final_text and is_final_model_text_event( - event, agent.name - ): + if is_final_model_text_event(event, agent.name): final_text_events.append(event) continue - yield event + # Partials go out immediately, even while a + # durable event is held back as the merge target. + # Parking them behind it would stall the live + # stream for the rest of the turn: a command's + # output and the final answer's deltas both arrive + # after the last durable event. Overtaking is safe + # because partials are never persisted + # (`BaseSessionService.append_event` returns early + # on them), so only the order among durable events + # is observable in session history, and that order + # is unchanged. + if event.partial: + yield event + continue + transfer_target = transfer_agent_name(event) if transfer_target: + if merge_target is not None: + yield merge_target + merge_target = None final_text_events.clear() + yield event async for transferred_event in run_transferred_agent( ctx, event, @@ -197,11 +240,27 @@ async def _pump_pi() -> None: return_exceptions=True, ) return + if merge_target is not None: + yield merge_target + merge_target = event await pump + except LlmCallsLimitExceededError as e: + # ADK raises this outside its on_model_error handling, + # so it must propagate rather than be turned into a + # model-error fallback. Leaving the `async with` blocks + # terminates the Pi subprocess, stopping the overrun. + # Nothing already streamed may be lost to the abort. + if merge_target is not None: + yield merge_target + merge_target = None + _emit_call_llm_telemetry( + ctx, runtime_call, _error_llm_response(e), call_llm_span + ) + raise except Exception as e: - if pump is not None and not pump.done(): - pump.cancel() - await asyncio.gather(pump, return_exceptions=True) + if merge_target is not None: + yield merge_target + merge_target = None fallback = await run_on_model_error_callbacks( agent, ctx, @@ -210,32 +269,201 @@ async def _pump_pi() -> None: runtime_call.model_response_event, ) if fallback is None: + _emit_call_llm_telemetry( + ctx, + runtime_call, + _error_llm_response(e), + call_llm_span, + ) raise + _emit_call_llm_telemetry( + ctx, runtime_call, fallback, call_llm_span + ) yield llm_response_to_event( runtime_call.llm_request, fallback, runtime_call.model_response_event, ) return - if final_text_events: - llm_response = final_events_to_llm_response(final_text_events) - llm_response = await run_after_model_callbacks( - agent, - ctx, - llm_response, - runtime_call.model_response_event, - ) - yield llm_response_to_event( - runtime_call.llm_request, - llm_response, - runtime_call.model_response_event, - ) + + # One merged response per turn, always: after-model callbacks must + # run on every turn (ADK does, and the harness collects token usage + # only through them), so this is not gated on there being text. + llm_response = final_events_to_llm_response(final_text_events) + # Exactly one usage carrier per turn: consumers sum + # `usage_metadata` across events without deduplicating. + usage_metadata = translator.build_turn_usage_metadata() + if usage_metadata is not None: + llm_response.usage_metadata = usage_metadata + llm_response = await run_after_model_callbacks( + agent, + ctx, + llm_response, + runtime_call.model_response_event, + ) + _emit_call_llm_telemetry(ctx, runtime_call, llm_response, call_llm_span) + event = llm_response_to_event( + runtime_call.llm_request, + llm_response, + runtime_call.model_response_event, + ) + if event.content and event.content.parts: + if merge_target is not None: + yield merge_target + merge_target = None + yield event + elif merge_target is not None: + # A tool-only turn: the merged event has no text, and a + # contentless, tool-free, non-partial event is a final response + # by `Event.is_final_response()` -- a spurious "the agent + # answered" marker on a turn that only did tool work. Dropping + # it whole, however, also threw away the `state_delta` model + # callbacks wrote through + # `CallbackContext(ctx, event_actions=model_response_event.actions)` + # and the turn's `usage_metadata`. Marking it partial does not + # rescue either: partial events are never persisted + # (`google/adk/sessions/base_session_service.py`). So the + # bookkeeping is folded onto the last event this turn actually + # emits -- an event that is persisted and is not a final + # response -- and the empty marker is never emitted. + merge_turn_bookkeeping(merge_target, event) + yield merge_target + merge_target = None + else: + # Pi answered nothing at all this turn (or only thought), so + # there is no durable event to fold onto and the bookkeeping + # would otherwise be lost outright. Emitting the empty event is + # then the lesser evil: it displaces no answer, because the turn + # produced none, and VeADK's two readers of "the final response" + # -- `maybe_save_output_to_state` and `base_evaluator` -- both + # require content before an event counts as one. + yield event finally: if tool_bundle is not None: await close_toolsets(tool_bundle.opened_toolsets) skill_bundle.close() + _end_span(call_llm_span) + +def _start_call_llm_span() -> "Span | None": + """Open the ADK-shaped ``call_llm`` span for one Pi invocation. -def _scope_event(event: Event, ctx: InvocationContext) -> None: + VeADK keys its whole model-telemetry chain off a span literally named + ``call_llm`` in ADK's tracer scope: the in-memory exporter indexes sessions + by it, the evaluator reads its prompt/completion attributes, and portal + metrics and the common model-span attributes are written from + :func:`veadk.tracing.telemetry.telemetry.trace_call_llm`. ADK opens that + span inside the LLM flow this runtime replaces, so the runtime must open it + itself. + + ``start_span`` is used rather than ``start_as_current_span``: ``run_async`` + is an async generator, so a context manager spanning its ``yield`` points + would attach the OTel context in one task resumption and detach it in + another, corrupting the context stack. Keeping the span non-current also + leaves tool spans as siblings of ``call_llm`` under ``invoke_agent``, which + is ADK's own shape. + + Returns: + Span | None: The started span, or ``None`` when tracing is unavailable. + """ + try: + from google.adk.telemetry.tracing import tracer + + return tracer.start_span("call_llm") + except Exception: # noqa: BLE001 + logger.warning("piagent_trace_span_start_failed") + return None + + +def _end_span(span: "Span | None") -> None: + """End a span without ever failing the turn.""" + if span is None: + return + try: + span.end() + except Exception: # noqa: BLE001 + logger.warning("piagent_trace_span_end_failed") + + +def _scope_event(event: "Event", ctx: "InvocationContext") -> None: event.branch = getattr(ctx, "branch", None) event.isolation_scope = getattr(ctx, "isolation_scope", None) + + +def _emit_call_llm_telemetry( + ctx: "InvocationContext", + runtime_call: RuntimeLlmCall, + llm_response: "LlmResponse", + span: "Span | None", +) -> None: + """Write one turn's model telemetry onto the ``call_llm`` span. + + Emitted exactly once per invocation, because the evaluator reads the first + span's prompt as the user input and the last span's completion as the final + answer, and the telemetry layer accumulates tokens per span. + + The span is made current only for this synchronous call, since portal + metrics derive the call duration from the current span's start time. + + Args: + ctx (InvocationContext): The invocation being served. + runtime_call (RuntimeLlmCall): The request built for this invocation. + llm_response (LlmResponse): The merged response for this turn. + span (Span | None): The owning ``call_llm`` span, if tracing is active. + """ + if span is None: + return + try: + from opentelemetry import trace as otel_trace + + from veadk.tracing.telemetry.telemetry import trace_call_llm + + with otel_trace.use_span(span, end_on_exit=False): + trace_call_llm( + ctx, + runtime_call.model_response_event.id, + runtime_call.llm_request, + llm_response, + span, + ) + except Exception: # noqa: BLE001 + logger.warning( + "piagent_trace_call_llm_failed invocation_id=%s", + getattr(ctx, "invocation_id", ""), + ) + + +def _charge_llm_call(ctx: "InvocationContext") -> None: + """Charge one backend model call to the invocation's ADK call budget. + + ADK enforces ``RunConfig.max_llm_calls`` solely from + ``InvocationContext.increment_llm_call_count``, which only its own + ``BaseLlmFlow`` calls. This runtime replaces that flow, so without this hook + ``max_llm_calls`` never fires for ``runtime="piagent"``. + + Pi owns its agent loop inside the binary, so unlike ADK -- which charges + *before* dispatching a call and therefore prevents the overrunning call -- + this can only charge a call Pi has already completed and reported. The + budget is therefore enforced one call late: the invocation is aborted once + the limit is passed, rather than stopped just short of it. + + Args: + ctx (InvocationContext): The invocation being served. + + Raises: + google.adk.agents.invocation_context.LlmCallsLimitExceededError: When + the invocation exceeds ``RunConfig.max_llm_calls``. + """ + increment = getattr(ctx, "increment_llm_call_count", None) + if callable(increment): + increment() + + +def _error_llm_response(error: BaseException) -> "LlmResponse": + """Build the response recorded on the span when a turn fails.""" + from google.adk.models.llm_response import LlmResponse + + return LlmResponse( + error_code=type(error).__name__, + error_message=str(error) or type(error).__name__, + ) diff --git a/veadk/runtime/piagent/translate.py b/veadk/runtime/piagent/translate.py index 02528f44c..8aff74723 100644 --- a/veadk/runtime/piagent/translate.py +++ b/veadk/runtime/piagent/translate.py @@ -116,6 +116,40 @@ def _system_instruction_text(value: Any) -> str: return str(value).strip() +# Pi's `Usage` counters, as published on assistant messages and tool results. +# `reasoning` is deliberately tracked but never mapped onto genai's +# `thoughts_token_count`: it is a *subset* of `output`, whereas genai treats +# thoughts as disjoint from candidates, so mapping it would double-count. +_USAGE_FIELDS = ( + "input", + "output", + "cacheRead", + "cacheWrite", + "reasoning", + "totalTokens", +) + + +def counts_as_model_call(event: dict[str, Any]) -> bool: + """Whether a raw Pi RPC event marks one completed backend model call. + + Pi's agent loop performs exactly one backend model call per iteration and + closes it with exactly one ``message_end`` carrying the assistant message. + ``message_end`` is also emitted for user and tool-result messages, so the + role check is what makes this a model-call boundary rather than a message + boundary. + + Args: + event (dict[str, Any]): One raw Pi RPC event. + + Returns: + bool: Whether the event closes one backend model call. + """ + if event.get("type") != "message_end": + return False + return _is_assistant_message(event.get("message")) + + def make_text_event( text: str, author: str, @@ -160,11 +194,16 @@ def __init__( self.author = author self.invocation_id = invocation_id self.bridged_tool_names = set(bridged_tool_names or ()) - self.emitted_text = False + self._emitted_texts: list[str] = [] + # Text already carried out on a tool-call event, awaiting the + # `message_end` that closes the round it belonged to. + self._carried_text: str | None = None self._thinking_parts: list[str] = [] self._text_parts: list[str] = [] + self._usage_totals: dict[str, int] = {} def event_to_adk_events(self, event: dict[str, Any]) -> list[Event]: + self._accumulate_usage(event) event_type = event.get("type") if event_type == "message_update": return self._message_update_to_events(event) @@ -185,7 +224,10 @@ def event_to_adk_events(self, event: dict[str, Any]) -> list[Event]: message = event.get("message") if _message_is_thinking(message): return [] - return self._flush_events(preferred_text=_message_text(message)) + return self._flush_events( + preferred_text=_message_text(message), + round_end=_is_assistant_message(message), + ) if event_type == "turn_end": return self._flush_events() if event_type == "agent_end": @@ -199,6 +241,105 @@ def event_to_adk_events(self, event: dict[str, Any]) -> list[Event]: def _is_bridged_tool_event(self, event: dict[str, Any]) -> bool: return str(event.get("toolName") or "") in self.bridged_tool_names + def _accumulate_usage(self, event: dict[str, Any]) -> None: + """Add one raw Pi event's token usage to this turn's running totals. + + Only sources that contribute *new* tokens are summed, mirroring Pi's own + accounting: + + - ``message_end`` for an assistant message -- one backend model call. + - ``turn_end.toolResults[]`` -- LLM work performed inside a tool (a + subagent), which no other event reports. + - ``compaction_end.result`` -- the call that produced a compaction + summary, which bypasses the agent loop. + + Deliberately excluded: ``message_update.usage`` is cumulative for the + in-flight message rather than incremental, ``turn_end.message`` repeats + the assistant message already counted at its ``message_end``, and + ``agent_end.messages[]`` replays the whole conversation. + + Args: + event (dict[str, Any]): One raw Pi RPC event. + """ + event_type = event.get("type") + if event_type == "message_end": + message = event.get("message") + if isinstance(message, dict) and message.get("role") == "assistant": + self._add_usage(message.get("usage")) + elif event_type == "turn_end": + results = event.get("toolResults") + if isinstance(results, list): + for result in results: + if isinstance(result, dict): + self._add_usage(result.get("usage")) + elif event_type == "compaction_end": + result = event.get("result") + if isinstance(result, dict): + self._add_usage(result.get("usage")) + + def _add_usage(self, usage: Any) -> None: + if not isinstance(usage, dict): + return + for key in _USAGE_FIELDS: + value = usage.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + self._usage_totals[key] = self._usage_totals.get(key, 0) + int(value) + + def build_turn_usage_metadata( + self, + ) -> types.GenerateContentResponseUsageMetadata | None: + """Map this turn's accumulated Pi usage onto genai usage metadata. + + Pi normalizes every provider to disjoint prompt-side counters: + ``input`` excludes cached tokens, which are reported separately as + ``cacheRead`` (a cache hit) and ``cacheWrite`` (a cache entry being + created); adapters for providers that fold cache into prompt tokens + subtract them explicitly. genai's ``prompt_token_count`` is instead the + whole prompt, with ``cached_content_token_count`` a *subset* of it, so + the three are summed into the prompt count and the cache-read figure is + also surfaced on its own. ``cacheWrite`` counts as prompt because those + tokens are part of the prompt the model processes: Pi's own cost model + likewise sizes the input side as ``input + cacheRead + cacheWrite``, + billing the three at different rates rather than treating any of them as + non-prompt. + + The total is the larger of Pi's reported ``totalTokens`` and the + component sum, because neither alone is right for every provider. Most + adapters compute ``totalTokens`` as exactly that sum, but some pass the + provider's figure through verbatim -- which can legitimately *exceed* + the sum, since genai's total also covers tool-use prompt tokens that Pi + never breaks out -- while at least one falls back to ``input + output`` + alone, which against a prompt count that includes cache tokens would + report a total smaller than its own parts. Taking the maximum keeps + whichever figure carries more information without ever violating + ``total >= prompt + candidates``. + + Returns: + google.genai.types.GenerateContentResponseUsageMetadata | None: The + turn's usage, or ``None`` when Pi reported no usable counter, so a + missing or malformed payload degrades to "no accounting" rather than + to zeroed accounting that would pollute token histograms. + """ + if not self._usage_totals: + return None + totals = self._usage_totals + prompt = ( + totals.get("input", 0) + + totals.get("cacheRead", 0) + + totals.get("cacheWrite", 0) + ) + candidates = totals.get("output", 0) + total = max(prompt + candidates, totals.get("totalTokens", 0)) + if not total: + return None + return types.GenerateContentResponseUsageMetadata( + prompt_token_count=prompt or None, + candidates_token_count=candidates or None, + total_token_count=total, + cached_content_token_count=totals.get("cacheRead") or None, + ) + def _message_update_to_events(self, event: dict[str, Any]) -> list[Event]: update = event.get("assistantMessageEvent") if not isinstance(update, dict): @@ -233,23 +374,60 @@ def _message_update_to_events(self, event: dict[str, Any]) -> list[Event]: raise RuntimeError(f"Pi assistant error: {reason}") return [] - def _flush_events(self, *, preferred_text: str = "") -> list[Event]: - if self.emitted_text: - self._thinking_parts.clear() - self._text_parts.clear() - return [] - + def _flush_events( + self, *, preferred_text: str = "", round_end: bool = False + ) -> list[Event]: + """Emit this round's durable assistant text, if it is new. + + Two different things re-announce text Pi has already reported, and they + need two different suppression rules -- a single "have I seen this text + before" set gets one of them wrong whichever way it is tuned: + + - A **round end** (an assistant ``message_end``, ``round_end=True``) + repeats only the preamble its own round already carried out on a + tool-call event, so it is matched against that one parked string. + Matching it against every text ever emitted instead loses a genuine + answer that happens to be byte-identical to an earlier preamble (a + model that says "Done." beside its tool call and "Done." again as the + answer): the answering round is suppressed, no final response is + produced at all, and ``output_key`` is never written. + - A **replay** (``turn_end`` / ``agent_end`` / ``agent_settled``, + ``round_end=False``) re-announces the last assistant message wholesale + once the turn is over. Nothing new can arrive after it, so it is + matched against everything already emitted. + + Neither may become a "have I emitted anything yet" latch: that made the + *first* round win, so on a turn whose tool call carried a text preamble + ("let me check the weather...") the preamble became the turn's answer + and the round that actually answered was dropped. + + Args: + preferred_text (str): Text Pi reported for the message, preferred + over the accumulated deltas when present. + round_end (bool): Whether this is an assistant ``message_end`` + closing one round, rather than an end-of-turn replay. + """ if preferred_text: text = preferred_text self._text_parts.clear() else: text = self._drain_text() + carried = self._carried_text + if round_end: + # This round is over, so its parked preamble can no longer be + # re-announced -- whether or not this `message_end` repeated it. + self._carried_text = None if not text: return [] + duplicate = (text == carried) if round_end else (text in self._emitted_texts) + if duplicate: + self._thinking_parts.clear() + self._text_parts.clear() + return [] parts = self._drain_pending_parts(include_text=False) parts.append(types.Part(text=text, thought=False)) - self.emitted_text = True + self._note_emitted(text) return [ make_model_event( parts, @@ -268,9 +446,18 @@ def _drain_pending_parts(self, *, include_text: bool = True) -> list[types.Part] text = self._drain_text() if text: parts.append(types.Part(text=text, thought=False)) + # This text is now persisted on the carrying event (a tool + # call), so the `message_end` closing this round must not emit + # it again as a standalone answer. + self._carried_text = text + self._note_emitted(text) return parts + def _note_emitted(self, text: str) -> None: + if text not in self._emitted_texts: + self._emitted_texts.append(text) + def _drain_thinking(self) -> str: text = "".join(self._thinking_parts).strip() self._thinking_parts.clear() @@ -366,6 +553,10 @@ def _last_assistant_text(messages: Any) -> str: return "" +def _is_assistant_message(message: Any) -> bool: + return isinstance(message, dict) and message.get("role") == "assistant" + + def _message_is_thinking(message: Any) -> bool: if not isinstance(message, dict): return False